2024-12-08 00:14:01,675 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 00:14:01,685 main DEBUG Took 0.008998 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 00:14:01,686 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 00:14:01,686 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 00:14:01,687 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 00:14:01,688 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,695 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 00:14:01,708 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,709 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,710 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,711 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,712 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,713 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,714 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,715 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,715 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,716 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,716 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,717 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,717 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,717 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,718 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,719 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 00:14:01,720 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,720 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 00:14:01,722 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 00:14:01,723 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 00:14:01,726 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 00:14:01,726 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 00:14:01,728 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 00:14:01,728 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 00:14:01,737 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 00:14:01,741 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 00:14:01,742 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 00:14:01,743 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 00:14:01,743 main DEBUG createAppenders(={Console}) 2024-12-08 00:14:01,744 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-08 00:14:01,745 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 00:14:01,745 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-08 00:14:01,746 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 00:14:01,746 main DEBUG OutputStream closed 2024-12-08 00:14:01,747 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 00:14:01,747 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 00:14:01,747 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-08 00:14:01,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 00:14:01,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 00:14:01,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 00:14:01,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 00:14:01,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 00:14:01,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 00:14:01,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 00:14:01,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 00:14:01,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 00:14:01,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 00:14:01,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 00:14:01,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 00:14:01,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 00:14:01,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 00:14:01,836 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 00:14:01,836 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 00:14:01,836 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 00:14:01,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 00:14:01,839 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 00:14:01,839 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-08 00:14:01,840 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 00:14:01,840 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-08T00:14:02,117 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042 2024-12-08 00:14:02,120 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 00:14:02,120 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T00:14:02,127 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-08T00:14:02,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T00:14:02,158 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/cluster_da41c416-9640-b4a5-e754-4cc4f92f286a, deleteOnExit=true 2024-12-08T00:14:02,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T00:14:02,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/test.cache.data in system properties and HBase conf 2024-12-08T00:14:02,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T00:14:02,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/hadoop.log.dir in system properties and HBase conf 2024-12-08T00:14:02,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T00:14:02,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T00:14:02,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T00:14:02,261 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T00:14:02,344 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T00:14:02,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T00:14:02,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T00:14:02,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T00:14:02,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T00:14:02,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T00:14:02,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T00:14:02,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T00:14:02,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T00:14:02,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T00:14:02,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/nfs.dump.dir in system properties and HBase conf 2024-12-08T00:14:02,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/java.io.tmpdir in system properties and HBase conf 2024-12-08T00:14:02,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T00:14:02,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T00:14:02,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T00:14:03,656 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T00:14:03,724 INFO [Time-limited test {}] log.Log(170): Logging initialized @2659ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T00:14:03,785 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T00:14:03,837 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T00:14:03,855 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T00:14:03,856 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T00:14:03,857 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T00:14:03,869 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T00:14:03,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/hadoop.log.dir/,AVAILABLE} 2024-12-08T00:14:03,873 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T00:14:04,042 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/java.io.tmpdir/jetty-localhost-35805-hadoop-hdfs-3_4_1-tests_jar-_-any-11382485434737395883/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T00:14:04,049 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35805} 2024-12-08T00:14:04,049 INFO [Time-limited test {}] server.Server(415): Started @2985ms 2024-12-08T00:14:04,694 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T00:14:04,702 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T00:14:04,705 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T00:14:04,705 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T00:14:04,705 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T00:14:04,706 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/hadoop.log.dir/,AVAILABLE} 2024-12-08T00:14:04,707 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T00:14:04,806 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/java.io.tmpdir/jetty-localhost-36267-hadoop-hdfs-3_4_1-tests_jar-_-any-9798851698377582699/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T00:14:04,807 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:36267} 2024-12-08T00:14:04,807 INFO [Time-limited test {}] server.Server(415): Started @3743ms 2024-12-08T00:14:04,851 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T00:14:06,084 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/cluster_da41c416-9640-b4a5-e754-4cc4f92f286a/data/data2/current/BP-1600602867-172.17.0.2-1733616842832/current, will proceed with Du for space computation calculation, 2024-12-08T00:14:06,084 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/cluster_da41c416-9640-b4a5-e754-4cc4f92f286a/data/data1/current/BP-1600602867-172.17.0.2-1733616842832/current, will proceed with Du for space computation calculation, 2024-12-08T00:14:06,108 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T00:14:06,164 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9eefbdb8c961d41 with lease ID 0x7d4539329dee0360: Processing first storage report for DS-a085cf9c-91da-4922-9dd6-b4549cd36da9 from datanode DatanodeRegistration(127.0.0.1:36767, datanodeUuid=99912f89-3e06-4776-a0b9-bcdf0d55620a, infoPort=46231, infoSecurePort=0, ipcPort=42261, storageInfo=lv=-57;cid=testClusterID;nsid=1395792235;c=1733616842832) 2024-12-08T00:14:06,167 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9eefbdb8c961d41 with lease ID 0x7d4539329dee0360: from storage DS-a085cf9c-91da-4922-9dd6-b4549cd36da9 node DatanodeRegistration(127.0.0.1:36767, datanodeUuid=99912f89-3e06-4776-a0b9-bcdf0d55620a, infoPort=46231, infoSecurePort=0, ipcPort=42261, storageInfo=lv=-57;cid=testClusterID;nsid=1395792235;c=1733616842832), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-12-08T00:14:06,167 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9eefbdb8c961d41 with lease ID 0x7d4539329dee0360: Processing first storage report for DS-67085db6-768e-4667-9312-d7d38f9ee8f5 from datanode DatanodeRegistration(127.0.0.1:36767, datanodeUuid=99912f89-3e06-4776-a0b9-bcdf0d55620a, infoPort=46231, infoSecurePort=0, ipcPort=42261, storageInfo=lv=-57;cid=testClusterID;nsid=1395792235;c=1733616842832) 2024-12-08T00:14:06,167 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9eefbdb8c961d41 with lease ID 0x7d4539329dee0360: from storage DS-67085db6-768e-4667-9312-d7d38f9ee8f5 node DatanodeRegistration(127.0.0.1:36767, datanodeUuid=99912f89-3e06-4776-a0b9-bcdf0d55620a, infoPort=46231, infoSecurePort=0, ipcPort=42261, storageInfo=lv=-57;cid=testClusterID;nsid=1395792235;c=1733616842832), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T00:14:06,243 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042 2024-12-08T00:14:06,378 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/cluster_da41c416-9640-b4a5-e754-4cc4f92f286a/zookeeper_0, clientPort=60060, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/cluster_da41c416-9640-b4a5-e754-4cc4f92f286a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/cluster_da41c416-9640-b4a5-e754-4cc4f92f286a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T00:14:06,390 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60060 2024-12-08T00:14:06,399 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T00:14:06,401 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T00:14:06,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741825_1001 (size=7) 2024-12-08T00:14:07,045 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329 with version=8 2024-12-08T00:14:07,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/hbase-staging 2024-12-08T00:14:07,121 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T00:14:07,355 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c1a8cf8d4cfa:0 server-side Connection retries=45 2024-12-08T00:14:07,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T00:14:07,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T00:14:07,371 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T00:14:07,371 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T00:14:07,371 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T00:14:07,517 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T00:14:07,581 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T00:14:07,590 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T00:14:07,595 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T00:14:07,618 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4813 (auto-detected) 2024-12-08T00:14:07,620 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T00:14:07,640 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46259 2024-12-08T00:14:07,665 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46259 connecting to ZooKeeper ensemble=127.0.0.1:60060 2024-12-08T00:14:07,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462590x0, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T00:14:07,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46259-0x10002c5ccce0000 connected 2024-12-08T00:14:07,869 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T00:14:07,872 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T00:14:07,882 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T00:14:07,886 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329, hbase.cluster.distributed=false 2024-12-08T00:14:07,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T00:14:07,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46259 2024-12-08T00:14:07,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46259 2024-12-08T00:14:07,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46259 2024-12-08T00:14:07,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46259 2024-12-08T00:14:07,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46259 2024-12-08T00:14:08,000 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c1a8cf8d4cfa:0 server-side Connection retries=45 2024-12-08T00:14:08,001 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T00:14:08,001 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T00:14:08,001 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T00:14:08,002 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T00:14:08,002 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T00:14:08,004 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T00:14:08,006 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T00:14:08,007 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36021 2024-12-08T00:14:08,008 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36021 connecting to ZooKeeper ensemble=127.0.0.1:60060 2024-12-08T00:14:08,009 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T00:14:08,012 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T00:14:08,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360210x0, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T00:14:08,025 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360210x0, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T00:14:08,025 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36021-0x10002c5ccce0001 connected 2024-12-08T00:14:08,028 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T00:14:08,035 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T00:14:08,037 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T00:14:08,042 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T00:14:08,043 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36021 2024-12-08T00:14:08,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36021 2024-12-08T00:14:08,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36021 2024-12-08T00:14:08,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36021 2024-12-08T00:14:08,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36021 2024-12-08T00:14:08,060 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c1a8cf8d4cfa:46259 2024-12-08T00:14:08,061 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:08,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T00:14:08,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T00:14:08,080 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:08,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:08,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T00:14:08,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:08,120 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T00:14:08,121 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c1a8cf8d4cfa,46259,1733616847193 from backup master directory 2024-12-08T00:14:08,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:08,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T00:14:08,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T00:14:08,130 WARN [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T00:14:08,130 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:08,133 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T00:14:08,134 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T00:14:08,186 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/hbase.id] with ID: 360fcdd9-b471-4f97-a388-64d212b7bcc8 2024-12-08T00:14:08,186 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/.tmp/hbase.id 2024-12-08T00:14:08,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741826_1002 (size=42) 2024-12-08T00:14:08,196 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/.tmp/hbase.id]:[hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/hbase.id] 2024-12-08T00:14:08,240 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T00:14:08,244 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T00:14:08,261 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-08T00:14:08,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:08,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:08,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741827_1003 (size=196) 2024-12-08T00:14:08,308 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:14:08,310 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T00:14:08,323 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:08,327 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T00:14:08,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741828_1004 (size=1189) 2024-12-08T00:14:08,779 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store 2024-12-08T00:14:08,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741829_1005 (size=34) 2024-12-08T00:14:09,206 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T00:14:09,210 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:14:09,211 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T00:14:09,211 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T00:14:09,211 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T00:14:09,212 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T00:14:09,212 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T00:14:09,213 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T00:14:09,213 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733616849211Disabling compacts and flushes for region at 1733616849211Disabling writes for close at 1733616849212 (+1 ms)Writing region close event to WAL at 1733616849213 (+1 ms)Closed at 1733616849213 2024-12-08T00:14:09,215 WARN [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/.initializing 2024-12-08T00:14:09,216 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/WALs/c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:09,223 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T00:14:09,235 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c1a8cf8d4cfa%2C46259%2C1733616847193, suffix=, logDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/WALs/c1a8cf8d4cfa,46259,1733616847193, archiveDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/oldWALs, maxLogs=10 2024-12-08T00:14:09,254 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/WALs/c1a8cf8d4cfa,46259,1733616847193/c1a8cf8d4cfa%2C46259%2C1733616847193.1733616849238, exclude list is [], retry=0 2024-12-08T00:14:09,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36767,DS-a085cf9c-91da-4922-9dd6-b4549cd36da9,DISK] 2024-12-08T00:14:09,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T00:14:09,301 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/WALs/c1a8cf8d4cfa,46259,1733616847193/c1a8cf8d4cfa%2C46259%2C1733616847193.1733616849238 2024-12-08T00:14:09,302 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46231:46231)] 2024-12-08T00:14:09,302 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:14:09,303 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:14:09,306 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,307 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,359 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T00:14:09,363 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:09,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T00:14:09,366 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,369 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T00:14:09,370 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:09,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:09,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T00:14:09,374 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:09,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:09,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,378 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T00:14:09,378 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:09,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:09,380 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,384 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,385 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,391 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,392 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,396 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T00:14:09,399 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T00:14:09,403 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T00:14:09,404 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60021857, jitterRate=-0.1056046336889267}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T00:14:09,409 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733616849317Initializing all the Stores at 1733616849319 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733616849319Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616849320 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616849320Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616849320Cleaning up temporary data from old regions at 1733616849392 (+72 ms)Region opened successfully at 1733616849409 (+17 ms) 2024-12-08T00:14:09,410 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T00:14:09,437 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d43f4bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c1a8cf8d4cfa/172.17.0.2:0 2024-12-08T00:14:09,462 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T00:14:09,471 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T00:14:09,471 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T00:14:09,474 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T00:14:09,475 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-08T00:14:09,479 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-08T00:14:09,479 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T00:14:09,500 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T00:14:09,508 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T00:14:09,550 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T00:14:09,554 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T00:14:09,556 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T00:14:09,561 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T00:14:09,563 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T00:14:09,567 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T00:14:09,571 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T00:14:09,573 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T00:14:09,582 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T00:14:09,600 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T00:14:09,612 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T00:14:09,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T00:14:09,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T00:14:09,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:09,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:09,628 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c1a8cf8d4cfa,46259,1733616847193, sessionid=0x10002c5ccce0000, setting cluster-up flag (Was=false) 2024-12-08T00:14:09,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:09,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:09,688 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T00:14:09,693 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:09,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:09,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:09,750 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T00:14:09,755 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:09,763 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T00:14:09,825 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T00:14:09,834 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T00:14:09,839 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T00:14:09,843 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c1a8cf8d4cfa,46259,1733616847193 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T00:14:09,850 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c1a8cf8d4cfa:0, corePoolSize=5, maxPoolSize=5 2024-12-08T00:14:09,850 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c1a8cf8d4cfa:0, corePoolSize=5, maxPoolSize=5 2024-12-08T00:14:09,850 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c1a8cf8d4cfa:0, corePoolSize=5, maxPoolSize=5 2024-12-08T00:14:09,850 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c1a8cf8d4cfa:0, corePoolSize=5, maxPoolSize=5 2024-12-08T00:14:09,851 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c1a8cf8d4cfa:0, corePoolSize=10, maxPoolSize=10 2024-12-08T00:14:09,851 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:09,851 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c1a8cf8d4cfa:0, corePoolSize=2, maxPoolSize=2 2024-12-08T00:14:09,851 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:09,852 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733616879852 2024-12-08T00:14:09,853 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(746): ClusterId : 360fcdd9-b471-4f97-a388-64d212b7bcc8 2024-12-08T00:14:09,854 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T00:14:09,855 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T00:14:09,855 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T00:14:09,857 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T00:14:09,857 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T00:14:09,859 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T00:14:09,859 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T00:14:09,860 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T00:14:09,860 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T00:14:09,861 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:09,863 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:09,863 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T00:14:09,864 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T00:14:09,866 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T00:14:09,866 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T00:14:09,868 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T00:14:09,868 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T00:14:09,868 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T00:14:09,868 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T00:14:09,870 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c1a8cf8d4cfa:0:becomeActiveMaster-HFileCleaner.large.0-1733616849870,5,FailOnTimeoutGroup] 2024-12-08T00:14:09,871 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c1a8cf8d4cfa:0:becomeActiveMaster-HFileCleaner.small.0-1733616849870,5,FailOnTimeoutGroup] 2024-12-08T00:14:09,871 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:09,871 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T00:14:09,872 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:09,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741831_1007 (size=1321) 2024-12-08T00:14:09,873 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:09,878 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T00:14:09,878 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72399cd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c1a8cf8d4cfa/172.17.0.2:0 2024-12-08T00:14:09,890 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c1a8cf8d4cfa:36021 2024-12-08T00:14:09,893 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T00:14:09,893 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T00:14:09,893 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T00:14:09,895 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(2659): reportForDuty to master=c1a8cf8d4cfa,46259,1733616847193 with port=36021, startcode=1733616847970 2024-12-08T00:14:09,905 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T00:14:09,937 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41271, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T00:14:09,942 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46259 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:09,944 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46259 {}] master.ServerManager(517): Registering regionserver=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:09,959 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329 2024-12-08T00:14:09,959 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38367 2024-12-08T00:14:09,959 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T00:14:09,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T00:14:09,972 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] zookeeper.ZKUtil(111): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:09,972 WARN [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T00:14:09,972 INFO [RS:0;c1a8cf8d4cfa:36021 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T00:14:09,973 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/WALs/c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:09,974 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c1a8cf8d4cfa,36021,1733616847970] 2024-12-08T00:14:09,997 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T00:14:10,008 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T00:14:10,012 INFO [RS:0;c1a8cf8d4cfa:36021 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T00:14:10,013 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,014 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T00:14:10,019 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T00:14:10,020 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,020 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,021 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,021 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,021 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,021 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,021 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c1a8cf8d4cfa:0, corePoolSize=2, maxPoolSize=2 2024-12-08T00:14:10,021 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,021 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,022 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,022 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,022 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,022 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c1a8cf8d4cfa:0, corePoolSize=1, maxPoolSize=1 2024-12-08T00:14:10,022 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c1a8cf8d4cfa:0, corePoolSize=3, maxPoolSize=3 2024-12-08T00:14:10,023 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0, corePoolSize=3, maxPoolSize=3 2024-12-08T00:14:10,024 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,024 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,024 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,024 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,024 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,024 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=c1a8cf8d4cfa,36021,1733616847970-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T00:14:10,040 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T00:14:10,041 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=c1a8cf8d4cfa,36021,1733616847970-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,042 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,042 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.Replication(171): c1a8cf8d4cfa,36021,1733616847970 started 2024-12-08T00:14:10,057 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:10,058 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1482): Serving as c1a8cf8d4cfa,36021,1733616847970, RpcServer on c1a8cf8d4cfa/172.17.0.2:36021, sessionid=0x10002c5ccce0001 2024-12-08T00:14:10,058 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T00:14:10,058 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:10,058 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c1a8cf8d4cfa,36021,1733616847970' 2024-12-08T00:14:10,059 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T00:14:10,059 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T00:14:10,060 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T00:14:10,060 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T00:14:10,060 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:10,060 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c1a8cf8d4cfa,36021,1733616847970' 2024-12-08T00:14:10,060 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T00:14:10,061 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T00:14:10,062 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T00:14:10,062 INFO [RS:0;c1a8cf8d4cfa:36021 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T00:14:10,062 INFO [RS:0;c1a8cf8d4cfa:36021 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T00:14:10,175 INFO [RS:0;c1a8cf8d4cfa:36021 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T00:14:10,179 INFO [RS:0;c1a8cf8d4cfa:36021 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c1a8cf8d4cfa%2C36021%2C1733616847970, suffix=, logDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/WALs/c1a8cf8d4cfa,36021,1733616847970, archiveDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/oldWALs, maxLogs=32 2024-12-08T00:14:10,191 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/WALs/c1a8cf8d4cfa,36021,1733616847970/c1a8cf8d4cfa%2C36021%2C1733616847970.1733616850180, exclude list is [], retry=0 2024-12-08T00:14:10,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36767,DS-a085cf9c-91da-4922-9dd6-b4549cd36da9,DISK] 2024-12-08T00:14:10,198 INFO [RS:0;c1a8cf8d4cfa:36021 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/WALs/c1a8cf8d4cfa,36021,1733616847970/c1a8cf8d4cfa%2C36021%2C1733616847970.1733616850180 2024-12-08T00:14:10,199 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46231:46231)] 2024-12-08T00:14:10,278 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T00:14:10,279 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329 2024-12-08T00:14:10,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741833_1009 (size=32) 2024-12-08T00:14:10,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:14:10,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T00:14:10,702 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T00:14:10,702 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:10,703 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T00:14:10,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T00:14:10,708 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T00:14:10,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:10,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T00:14:10,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T00:14:10,712 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T00:14:10,712 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:10,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T00:14:10,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T00:14:10,716 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T00:14:10,716 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:10,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T00:14:10,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T00:14:10,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740 2024-12-08T00:14:10,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740 2024-12-08T00:14:10,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T00:14:10,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T00:14:10,723 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:14:10,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T00:14:10,729 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T00:14:10,730 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65126238, jitterRate=-0.029543429613113403}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:14:10,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733616850694Initializing all the Stores at 1733616850697 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733616850697Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733616850697Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616850697Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733616850697Cleaning up temporary data from old regions at 1733616850722 (+25 ms)Region opened successfully at 1733616850731 (+9 ms) 2024-12-08T00:14:10,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T00:14:10,732 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T00:14:10,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T00:14:10,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T00:14:10,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T00:14:10,733 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T00:14:10,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733616850732Disabling compacts and flushes for region at 1733616850732Disabling writes for close at 1733616850732Writing region close event to WAL at 1733616850733 (+1 ms)Closed at 1733616850733 2024-12-08T00:14:10,736 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T00:14:10,737 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T00:14:10,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T00:14:10,749 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T00:14:10,751 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T00:14:10,908 DEBUG [c1a8cf8d4cfa:46259 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T00:14:10,918 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:10,923 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c1a8cf8d4cfa,36021,1733616847970, state=OPENING 2024-12-08T00:14:10,971 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T00:14:10,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:10,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:14:10,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T00:14:10,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T00:14:10,987 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T00:14:10,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:14:11,166 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T00:14:11,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44463, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T00:14:11,180 INFO [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T00:14:11,181 INFO [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T00:14:11,181 INFO [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T00:14:11,185 INFO [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c1a8cf8d4cfa%2C36021%2C1733616847970.meta, suffix=.meta, logDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/WALs/c1a8cf8d4cfa,36021,1733616847970, archiveDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/oldWALs, maxLogs=32 2024-12-08T00:14:11,201 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/WALs/c1a8cf8d4cfa,36021,1733616847970/c1a8cf8d4cfa%2C36021%2C1733616847970.meta.1733616851187.meta, exclude list is [], retry=0 2024-12-08T00:14:11,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36767,DS-a085cf9c-91da-4922-9dd6-b4549cd36da9,DISK] 2024-12-08T00:14:11,207 INFO [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/WALs/c1a8cf8d4cfa,36021,1733616847970/c1a8cf8d4cfa%2C36021%2C1733616847970.meta.1733616851187.meta 2024-12-08T00:14:11,207 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46231:46231)] 2024-12-08T00:14:11,208 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:14:11,210 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T00:14:11,213 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T00:14:11,219 INFO [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T00:14:11,223 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T00:14:11,224 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:14:11,224 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T00:14:11,225 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T00:14:11,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T00:14:11,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T00:14:11,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:11,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T00:14:11,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T00:14:11,232 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T00:14:11,232 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:11,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T00:14:11,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T00:14:11,235 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T00:14:11,235 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:11,236 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T00:14:11,236 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T00:14:11,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T00:14:11,237 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:11,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T00:14:11,238 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T00:14:11,240 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740 2024-12-08T00:14:11,242 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740 2024-12-08T00:14:11,244 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T00:14:11,244 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T00:14:11,246 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:14:11,250 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T00:14:11,252 INFO [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75447069, jitterRate=0.1242489367723465}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:14:11,252 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T00:14:11,254 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733616851225Writing region info on filesystem at 1733616851225Initializing all the Stores at 1733616851227 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733616851227Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733616851227Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616851228 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733616851228Cleaning up temporary data from old regions at 1733616851245 (+17 ms)Running coprocessor post-open hooks at 1733616851252 (+7 ms)Region opened successfully at 1733616851254 (+2 ms) 2024-12-08T00:14:11,261 INFO [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733616851159 2024-12-08T00:14:11,275 DEBUG [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T00:14:11,276 INFO [RS_OPEN_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T00:14:11,277 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:11,280 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c1a8cf8d4cfa,36021,1733616847970, state=OPEN 2024-12-08T00:14:11,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T00:14:11,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T00:14:11,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T00:14:11,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T00:14:11,358 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:11,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T00:14:11,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c1a8cf8d4cfa,36021,1733616847970 in 368 msec 2024-12-08T00:14:11,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T00:14:11,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 625 msec 2024-12-08T00:14:11,373 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T00:14:11,373 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T00:14:11,390 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:11,391 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:11,408 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:11,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50405, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:11,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6380 sec 2024-12-08T00:14:11,431 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733616851431, completionTime=-1 2024-12-08T00:14:11,434 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-08T00:14:11,434 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T00:14:11,455 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-08T00:14:11,455 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733616911455 2024-12-08T00:14:11,455 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733616971455 2024-12-08T00:14:11,455 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 20 msec 2024-12-08T00:14:11,457 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c1a8cf8d4cfa,46259,1733616847193-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:11,457 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c1a8cf8d4cfa,46259,1733616847193-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:11,457 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c1a8cf8d4cfa,46259,1733616847193-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:11,459 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c1a8cf8d4cfa:46259, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:11,459 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:11,460 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:11,464 DEBUG [master/c1a8cf8d4cfa:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T00:14:11,499 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.368sec 2024-12-08T00:14:11,500 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T00:14:11,502 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T00:14:11,503 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T00:14:11,503 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T00:14:11,503 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T00:14:11,504 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c1a8cf8d4cfa,46259,1733616847193-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T00:14:11,505 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c1a8cf8d4cfa,46259,1733616847193-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T00:14:11,513 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T00:14:11,514 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T00:14:11,514 INFO [master/c1a8cf8d4cfa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c1a8cf8d4cfa,46259,1733616847193-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T00:14:11,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:11,563 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T00:14:11,564 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T00:14:11,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:11,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:11,575 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:11,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:11,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424b8b64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:11,585 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:11,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:11,588 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:11,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:11,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:11,624 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=321, ProcessCount=11, AvailableMemoryMB=8983 2024-12-08T00:14:11,632 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T00:14:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:11,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@46b992bd 2024-12-08T00:14:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T00:14:11,644 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T00:14:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T00:14:11,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:14:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:11,661 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T00:14:11,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-08T00:14:11,663 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:11,665 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T00:14:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T00:14:11,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741835_1011 (size=1061) 2024-12-08T00:14:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T00:14:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T00:14:12,080 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329 2024-12-08T00:14:12,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741836_1012 (size=53) 2024-12-08T00:14:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T00:14:12,494 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:14:12,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9938c46f9ca7a3abf34703707edea068, disabling compactions & flushes 2024-12-08T00:14:12,495 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:12,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:12,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. after waiting 0 ms 2024-12-08T00:14:12,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:12,496 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:12,496 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9938c46f9ca7a3abf34703707edea068: Waiting for close lock at 1733616852495Disabling compacts and flushes for region at 1733616852495Disabling writes for close at 1733616852495Writing region close event to WAL at 1733616852496 (+1 ms)Closed at 1733616852496 2024-12-08T00:14:12,501 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T00:14:12,506 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733616852501"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733616852501"}]},"ts":"1733616852501"} 2024-12-08T00:14:12,510 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T00:14:12,512 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T00:14:12,515 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616852512"}]},"ts":"1733616852512"} 2024-12-08T00:14:12,519 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T00:14:12,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9938c46f9ca7a3abf34703707edea068, ASSIGN}] 2024-12-08T00:14:12,522 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9938c46f9ca7a3abf34703707edea068, ASSIGN 2024-12-08T00:14:12,524 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9938c46f9ca7a3abf34703707edea068, ASSIGN; state=OFFLINE, location=c1a8cf8d4cfa,36021,1733616847970; forceNewPlan=false, retain=false 2024-12-08T00:14:12,675 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9938c46f9ca7a3abf34703707edea068, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:12,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9938c46f9ca7a3abf34703707edea068, ASSIGN because future has completed 2024-12-08T00:14:12,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:14:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T00:14:12,841 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:12,841 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:14:12,841 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,842 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:14:12,842 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,842 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,845 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,851 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:14:12,851 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9938c46f9ca7a3abf34703707edea068 columnFamilyName A 2024-12-08T00:14:12,852 DEBUG [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:12,853 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] regionserver.HStore(327): Store=9938c46f9ca7a3abf34703707edea068/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:12,853 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,856 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:14:12,856 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9938c46f9ca7a3abf34703707edea068 columnFamilyName B 2024-12-08T00:14:12,856 DEBUG [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:12,857 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] regionserver.HStore(327): Store=9938c46f9ca7a3abf34703707edea068/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:12,857 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,859 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:14:12,860 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9938c46f9ca7a3abf34703707edea068 columnFamilyName C 2024-12-08T00:14:12,860 DEBUG [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:12,861 INFO [StoreOpener-9938c46f9ca7a3abf34703707edea068-1 {}] regionserver.HStore(327): Store=9938c46f9ca7a3abf34703707edea068/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:12,861 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:12,861 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,863 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,864 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,866 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,866 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,867 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:14:12,869 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,872 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T00:14:12,873 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9938c46f9ca7a3abf34703707edea068; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62681769, jitterRate=-0.06596885621547699}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:14:12,873 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:12,874 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9938c46f9ca7a3abf34703707edea068: Running coprocessor pre-open hook at 1733616852842Writing region info on filesystem at 1733616852842Initializing all the Stores at 1733616852845 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616852845Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616852845Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616852845Cleaning up temporary data from old regions at 1733616852866 (+21 ms)Running coprocessor post-open hooks at 1733616852874 (+8 ms)Region opened successfully at 1733616852874 2024-12-08T00:14:12,877 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., pid=6, masterSystemTime=1733616852834 2024-12-08T00:14:12,880 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:12,880 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:12,881 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9938c46f9ca7a3abf34703707edea068, regionState=OPEN, openSeqNum=2, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:12,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:14:12,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T00:14:12,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 in 207 msec 2024-12-08T00:14:12,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T00:14:12,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9938c46f9ca7a3abf34703707edea068, ASSIGN in 371 msec 2024-12-08T00:14:12,897 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T00:14:12,897 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616852897"}]},"ts":"1733616852897"} 2024-12-08T00:14:12,900 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T00:14:12,902 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T00:14:12,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2490 sec 2024-12-08T00:14:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T00:14:13,824 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:13,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,835 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,838 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,840 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,846 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T00:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@da7d497 2024-12-08T00:14:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T00:14:13,852 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T00:14:13,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,858 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,862 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,865 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,869 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,874 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,878 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,881 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,884 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7ca9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,888 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b21b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,890 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd327f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,894 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,898 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b096a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,901 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,903 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,907 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:13,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:13,914 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c994daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:13,917 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:13,922 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,922 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,922 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,922 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,924 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,925 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:13,925 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,926 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,929 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-08T00:14:13,933 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:13,935 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T00:14:13,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:14:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:14:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:14:13,942 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:14:13,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:14:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:14:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:14:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:14:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:14:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:14,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:14:14,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:14,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:14,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:14,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:14,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:14,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T00:14:14,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T00:14:14,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/93e92ba626c142b2b19df9069ffab0cc is 50, key is test_row_0/A:col10/1733616853999/Put/seqid=0 2024-12-08T00:14:14,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741837_1013 (size=12001) 2024-12-08T00:14:14,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616854094, max timestamp=1733616854170Num uniques 859; 2024-12-08T00:14:14,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616854094, max timestamp=1733616854170Num uniques 859; 2024-12-08T00:14:14,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:14,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:14,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616854094, max timestamp=1733616854170Num uniques 859; 2024-12-08T00:14:14,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T00:14:14,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616914244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616914249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616914244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616914253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616914259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T00:14:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616914402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616914402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616914412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616914412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616914413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T00:14:14,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:14,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/93e92ba626c142b2b19df9069ffab0cc 2024-12-08T00:14:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T00:14:14,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T00:14:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:14,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616914622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616914623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616914623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616914623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616914623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/98e6f38c9b0e46098eddeb92e0c39e3d is 50, key is test_row_0/B:col10/1733616853999/Put/seqid=0 2024-12-08T00:14:14,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741838_1014 (size=12001) 2024-12-08T00:14:14,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/98e6f38c9b0e46098eddeb92e0c39e3d 2024-12-08T00:14:14,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7b6b850f7515469d82320d0938787310 is 50, key is test_row_0/C:col10/1733616853999/Put/seqid=0 2024-12-08T00:14:14,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741839_1015 (size=12001) 2024-12-08T00:14:14,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T00:14:14,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:14,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T00:14:14,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:14,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:14,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:14,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616914942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616914942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616914943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616914943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616914943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T00:14:15,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T00:14:15,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:15,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:15,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:15,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:15,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:15,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7b6b850f7515469d82320d0938787310 2024-12-08T00:14:15,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/93e92ba626c142b2b19df9069ffab0cc as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/93e92ba626c142b2b19df9069ffab0cc 2024-12-08T00:14:15,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/93e92ba626c142b2b19df9069ffab0cc, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T00:14:15,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/98e6f38c9b0e46098eddeb92e0c39e3d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/98e6f38c9b0e46098eddeb92e0c39e3d 2024-12-08T00:14:15,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/98e6f38c9b0e46098eddeb92e0c39e3d, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T00:14:15,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7b6b850f7515469d82320d0938787310 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7b6b850f7515469d82320d0938787310 2024-12-08T00:14:15,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7b6b850f7515469d82320d0938787310, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T00:14:15,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T00:14:15,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:15,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:15,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:15,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:15,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:15,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 9938c46f9ca7a3abf34703707edea068 in 1232ms, sequenceid=13, compaction requested=false 2024-12-08T00:14:15,252 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T00:14:15,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:15,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T00:14:15,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:15,396 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-08T00:14:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:15,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/f805deadf9ba4654b033e4b6caba8c10 is 50, key is test_row_0/A:col10/1733616854247/Put/seqid=0 2024-12-08T00:14:15,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741840_1016 (size=12001) 2024-12-08T00:14:15,441 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/f805deadf9ba4654b033e4b6caba8c10 2024-12-08T00:14:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:15,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/a2eca89c463d4ec29696725b588993db is 50, key is test_row_0/B:col10/1733616854247/Put/seqid=0 2024-12-08T00:14:15,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741841_1017 (size=12001) 2024-12-08T00:14:15,499 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/a2eca89c463d4ec29696725b588993db 2024-12-08T00:14:15,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616915493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616915499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616915507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/5cb139be12ca4b938e71e3ed241d91af is 50, key is test_row_0/C:col10/1733616854247/Put/seqid=0 2024-12-08T00:14:15,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616915514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741842_1018 (size=12001) 2024-12-08T00:14:15,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/5cb139be12ca4b938e71e3ed241d91af 2024-12-08T00:14:15,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616915520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/f805deadf9ba4654b033e4b6caba8c10 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f805deadf9ba4654b033e4b6caba8c10 2024-12-08T00:14:15,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f805deadf9ba4654b033e4b6caba8c10, entries=150, sequenceid=48, filesize=11.7 K 2024-12-08T00:14:15,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/a2eca89c463d4ec29696725b588993db as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a2eca89c463d4ec29696725b588993db 2024-12-08T00:14:15,609 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a2eca89c463d4ec29696725b588993db, entries=150, sequenceid=48, filesize=11.7 K 2024-12-08T00:14:15,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/5cb139be12ca4b938e71e3ed241d91af as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/5cb139be12ca4b938e71e3ed241d91af 2024-12-08T00:14:15,628 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/5cb139be12ca4b938e71e3ed241d91af, entries=150, sequenceid=48, filesize=11.7 K 2024-12-08T00:14:15,630 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 9938c46f9ca7a3abf34703707edea068 in 234ms, sequenceid=48, compaction requested=false 2024-12-08T00:14:15,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:15,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:15,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-08T00:14:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-08T00:14:15,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T00:14:15,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6910 sec 2024-12-08T00:14:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:15,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:14:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:15,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7210 sec 2024-12-08T00:14:15,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/5af0a7f6293e4cbf9c6572f5e1febad3 is 50, key is test_row_0/A:col10/1733616855645/Put/seqid=0 2024-12-08T00:14:15,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741843_1019 (size=14341) 2024-12-08T00:14:15,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/5af0a7f6293e4cbf9c6572f5e1febad3 2024-12-08T00:14:15,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/253df10505ed4868942279b4ab2335f0 is 50, key is test_row_0/B:col10/1733616855645/Put/seqid=0 2024-12-08T00:14:15,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616915720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616915735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616915735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616915737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616915740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741844_1020 (size=12001) 2024-12-08T00:14:15,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/253df10505ed4868942279b4ab2335f0 2024-12-08T00:14:15,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/437b338517b445db8d11575d10713141 is 50, key is test_row_0/C:col10/1733616855645/Put/seqid=0 2024-12-08T00:14:15,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741845_1021 (size=12001) 2024-12-08T00:14:15,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/437b338517b445db8d11575d10713141 2024-12-08T00:14:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/5af0a7f6293e4cbf9c6572f5e1febad3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/5af0a7f6293e4cbf9c6572f5e1febad3 2024-12-08T00:14:15,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/5af0a7f6293e4cbf9c6572f5e1febad3, entries=200, sequenceid=62, filesize=14.0 K 2024-12-08T00:14:15,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616915851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616915851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616915852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/253df10505ed4868942279b4ab2335f0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/253df10505ed4868942279b4ab2335f0 2024-12-08T00:14:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616915852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616915852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:15,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/253df10505ed4868942279b4ab2335f0, entries=150, sequenceid=62, filesize=11.7 K 2024-12-08T00:14:15,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/437b338517b445db8d11575d10713141 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/437b338517b445db8d11575d10713141 2024-12-08T00:14:15,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/437b338517b445db8d11575d10713141, entries=150, sequenceid=62, filesize=11.7 K 2024-12-08T00:14:15,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9938c46f9ca7a3abf34703707edea068 in 244ms, sequenceid=62, compaction requested=true 2024-12-08T00:14:15,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:15,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:15,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:15,901 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:15,901 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:15,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:15,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:15,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:15,908 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:15,910 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/B is initiating minor compaction (all files) 2024-12-08T00:14:15,911 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/B in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:15,911 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/98e6f38c9b0e46098eddeb92e0c39e3d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a2eca89c463d4ec29696725b588993db, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/253df10505ed4868942279b4ab2335f0] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=35.2 K 2024-12-08T00:14:15,916 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 98e6f38c9b0e46098eddeb92e0c39e3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733616853984 2024-12-08T00:14:15,917 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a2eca89c463d4ec29696725b588993db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733616854244 2024-12-08T00:14:15,918 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 253df10505ed4868942279b4ab2335f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733616855492 2024-12-08T00:14:15,921 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:15,921 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/A is initiating minor compaction (all files) 2024-12-08T00:14:15,922 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/A in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:15,922 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/93e92ba626c142b2b19df9069ffab0cc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f805deadf9ba4654b033e4b6caba8c10, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/5af0a7f6293e4cbf9c6572f5e1febad3] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=37.4 K 2024-12-08T00:14:15,925 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e92ba626c142b2b19df9069ffab0cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733616853984 2024-12-08T00:14:15,926 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting f805deadf9ba4654b033e4b6caba8c10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733616854244 2024-12-08T00:14:15,928 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af0a7f6293e4cbf9c6572f5e1febad3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733616855492 2024-12-08T00:14:15,951 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#B#compaction#9 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:15,953 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/e944790b1e1843ed81465021c1d6ef29 is 50, key is test_row_0/B:col10/1733616855645/Put/seqid=0 2024-12-08T00:14:15,966 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#A#compaction#10 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:15,967 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/e6c496075ee74fcc9e340228b3b1da4f is 50, key is test_row_0/A:col10/1733616855645/Put/seqid=0 2024-12-08T00:14:15,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741846_1022 (size=12104) 2024-12-08T00:14:15,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741847_1023 (size=12104) 2024-12-08T00:14:15,992 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T00:14:15,997 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/e6c496075ee74fcc9e340228b3b1da4f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e6c496075ee74fcc9e340228b3b1da4f 2024-12-08T00:14:16,020 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/A of 9938c46f9ca7a3abf34703707edea068 into e6c496075ee74fcc9e340228b3b1da4f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:16,020 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:16,024 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/A, priority=13, startTime=1733616855896; duration=0sec 2024-12-08T00:14:16,024 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:16,025 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:A 2024-12-08T00:14:16,025 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:16,028 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:16,028 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/C is initiating minor compaction (all files) 2024-12-08T00:14:16,028 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/C in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:16,028 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7b6b850f7515469d82320d0938787310, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/5cb139be12ca4b938e71e3ed241d91af, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/437b338517b445db8d11575d10713141] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=35.2 K 2024-12-08T00:14:16,029 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6b850f7515469d82320d0938787310, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733616853984 2024-12-08T00:14:16,030 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb139be12ca4b938e71e3ed241d91af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733616854244 2024-12-08T00:14:16,032 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437b338517b445db8d11575d10713141, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733616855492 2024-12-08T00:14:16,062 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#C#compaction#11 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:16,063 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7392da633af3454f9a7ee5e07b802f24 is 50, key is test_row_0/C:col10/1733616855645/Put/seqid=0 2024-12-08T00:14:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:16,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T00:14:16,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:16,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:16,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:16,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:16,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:16,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:16,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741848_1024 (size=12104) 2024-12-08T00:14:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T00:14:16,082 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:16,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/68c567277f2440eba58cbf472bc2ae50 is 50, key is test_row_0/A:col10/1733616855736/Put/seqid=0 2024-12-08T00:14:16,084 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-08T00:14:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T00:14:16,092 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:16,094 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:16,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:16,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616916088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616916094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616916094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616916103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616916103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741849_1025 (size=12001) 2024-12-08T00:14:16,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/68c567277f2440eba58cbf472bc2ae50 2024-12-08T00:14:16,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/2b2bbf1ca3714dc9807104173671f625 is 50, key is test_row_0/B:col10/1733616855736/Put/seqid=0 2024-12-08T00:14:16,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741850_1026 (size=12001) 2024-12-08T00:14:16,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/2b2bbf1ca3714dc9807104173671f625 2024-12-08T00:14:16,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/125d6532c359489c86dbef983f7c9191 is 50, key is test_row_0/C:col10/1733616855736/Put/seqid=0 2024-12-08T00:14:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T00:14:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616916212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616916211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616916212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741851_1027 (size=12001) 2024-12-08T00:14:16,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/125d6532c359489c86dbef983f7c9191 2024-12-08T00:14:16,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616916221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616916222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/68c567277f2440eba58cbf472bc2ae50 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/68c567277f2440eba58cbf472bc2ae50 2024-12-08T00:14:16,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T00:14:16,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:16,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:16,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:16,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:16,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:16,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/68c567277f2440eba58cbf472bc2ae50, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T00:14:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/2b2bbf1ca3714dc9807104173671f625 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/2b2bbf1ca3714dc9807104173671f625 2024-12-08T00:14:16,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/2b2bbf1ca3714dc9807104173671f625, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T00:14:16,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/125d6532c359489c86dbef983f7c9191 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/125d6532c359489c86dbef983f7c9191 2024-12-08T00:14:16,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/125d6532c359489c86dbef983f7c9191, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T00:14:16,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9938c46f9ca7a3abf34703707edea068 in 233ms, sequenceid=83, compaction requested=false 2024-12-08T00:14:16,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:16,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T00:14:16,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:16,409 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T00:14:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T00:14:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:16,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8968646c7a8841939998cbc5d3a8b590 is 50, key is test_row_0/A:col10/1733616856093/Put/seqid=0 2024-12-08T00:14:16,443 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/e944790b1e1843ed81465021c1d6ef29 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e944790b1e1843ed81465021c1d6ef29 2024-12-08T00:14:16,462 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/B of 9938c46f9ca7a3abf34703707edea068 into e944790b1e1843ed81465021c1d6ef29(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:16,462 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:16,462 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/B, priority=13, startTime=1733616855900; duration=0sec 2024-12-08T00:14:16,462 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:16,462 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:B 2024-12-08T00:14:16,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741852_1028 (size=12001) 2024-12-08T00:14:16,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616916461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,475 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8968646c7a8841939998cbc5d3a8b590 2024-12-08T00:14:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616916469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616916471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616856457, max timestamp=1733616856472Num uniques 859; 2024-12-08T00:14:16,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:16,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616856457, max timestamp=1733616856472Num uniques 859; 2024-12-08T00:14:16,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:16,497 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7392da633af3454f9a7ee5e07b802f24 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7392da633af3454f9a7ee5e07b802f24 2024-12-08T00:14:16,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616856457, max timestamp=1733616856487Num uniques 859; 2024-12-08T00:14:16,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/1191795e83b64d8ea14cf1fb8692291d is 50, key is test_row_0/B:col10/1733616856093/Put/seqid=0 2024-12-08T00:14:16,519 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/C of 9938c46f9ca7a3abf34703707edea068 into 7392da633af3454f9a7ee5e07b802f24(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:16,519 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:16,519 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/C, priority=13, startTime=1733616855903; duration=0sec 2024-12-08T00:14:16,519 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:16,519 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:C 2024-12-08T00:14:16,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741853_1029 (size=12001) 2024-12-08T00:14:16,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616916561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616916560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616916581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616916581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616916591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616916671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616916681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T00:14:16,735 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T00:14:16,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616916791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616916791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616916801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616916881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616916891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:16,930 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/1191795e83b64d8ea14cf1fb8692291d 2024-12-08T00:14:16,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/c630d29f70f24b9ca2cdf8fd1cf1143c is 50, key is test_row_0/C:col10/1733616856093/Put/seqid=0 2024-12-08T00:14:16,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741854_1030 (size=12001) 2024-12-08T00:14:16,989 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/c630d29f70f24b9ca2cdf8fd1cf1143c 2024-12-08T00:14:17,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8968646c7a8841939998cbc5d3a8b590 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8968646c7a8841939998cbc5d3a8b590 2024-12-08T00:14:17,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8968646c7a8841939998cbc5d3a8b590, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T00:14:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/1191795e83b64d8ea14cf1fb8692291d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1191795e83b64d8ea14cf1fb8692291d 2024-12-08T00:14:17,034 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1191795e83b64d8ea14cf1fb8692291d, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T00:14:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/c630d29f70f24b9ca2cdf8fd1cf1143c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c630d29f70f24b9ca2cdf8fd1cf1143c 2024-12-08T00:14:17,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c630d29f70f24b9ca2cdf8fd1cf1143c, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T00:14:17,050 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 9938c46f9ca7a3abf34703707edea068 in 641ms, sequenceid=95, compaction requested=true 2024-12-08T00:14:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-08T00:14:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-08T00:14:17,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-08T00:14:17,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 959 msec 2024-12-08T00:14:17,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 979 msec 2024-12-08T00:14:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:17,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T00:14:17,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733616856549, max timestamp=1733616857114Num uniques 1018; 2024-12-08T00:14:17,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T00:14:17,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:17,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:17,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:17,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T00:14:17,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:17,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T00:14:17,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/4419cf09051b40d4b3027813c63b19cf is 50, key is test_row_0/A:col10/1733616856557/Put/seqid=0 2024-12-08T00:14:17,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616917159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616917167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616917173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616917191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741855_1031 (size=14541) 2024-12-08T00:14:17,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616917201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T00:14:17,234 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:17,234 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-08T00:14:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T00:14:17,239 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:17,241 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:17,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:17,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616917281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616917281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616917281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T00:14:17,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:17,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:17,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616917501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616917501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616917501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:17,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T00:14:17,576 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T00:14:17,577 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T00:14:17,579 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T00:14:17,579 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T00:14:17,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T00:14:17,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T00:14:17,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T00:14:17,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T00:14:17,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/4419cf09051b40d4b3027813c63b19cf 2024-12-08T00:14:17,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/ba529cc32e7d408b9bb69cc5698d17ad is 50, key is test_row_0/B:col10/1733616856557/Put/seqid=0 2024-12-08T00:14:17,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741856_1032 (size=12151) 2024-12-08T00:14:17,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/ba529cc32e7d408b9bb69cc5698d17ad 2024-12-08T00:14:17,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:17,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616917711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/96f829f08229456391a1970bb37a2091 is 50, key is test_row_0/C:col10/1733616856557/Put/seqid=0 2024-12-08T00:14:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616917732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741857_1033 (size=12151) 2024-12-08T00:14:17,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/96f829f08229456391a1970bb37a2091 2024-12-08T00:14:17,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/4419cf09051b40d4b3027813c63b19cf as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4419cf09051b40d4b3027813c63b19cf 2024-12-08T00:14:17,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4419cf09051b40d4b3027813c63b19cf, entries=200, sequenceid=135, filesize=14.2 K 2024-12-08T00:14:17,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/ba529cc32e7d408b9bb69cc5698d17ad as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ba529cc32e7d408b9bb69cc5698d17ad 2024-12-08T00:14:17,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ba529cc32e7d408b9bb69cc5698d17ad, entries=150, sequenceid=135, filesize=11.9 K 2024-12-08T00:14:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/96f829f08229456391a1970bb37a2091 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/96f829f08229456391a1970bb37a2091 2024-12-08T00:14:17,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/96f829f08229456391a1970bb37a2091, entries=150, sequenceid=135, filesize=11.9 K 2024-12-08T00:14:17,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~292.16 KB/299168, currentSize=73.80 KB/75570 for 9938c46f9ca7a3abf34703707edea068 in 690ms, sequenceid=135, compaction requested=true 2024-12-08T00:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:17,805 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:17,805 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:17,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:17,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:17,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:17,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:17,808 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50647 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:17,808 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/A is initiating minor compaction (all files) 2024-12-08T00:14:17,808 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/A in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,808 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e6c496075ee74fcc9e340228b3b1da4f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/68c567277f2440eba58cbf472bc2ae50, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8968646c7a8841939998cbc5d3a8b590, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4419cf09051b40d4b3027813c63b19cf] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=49.5 K 2024-12-08T00:14:17,808 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:17,809 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/B is initiating minor compaction (all files) 2024-12-08T00:14:17,809 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/B in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,809 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e944790b1e1843ed81465021c1d6ef29, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/2b2bbf1ca3714dc9807104173671f625, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1191795e83b64d8ea14cf1fb8692291d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ba529cc32e7d408b9bb69cc5698d17ad] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=47.1 K 2024-12-08T00:14:17,810 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e944790b1e1843ed81465021c1d6ef29, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733616855492 2024-12-08T00:14:17,811 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2bbf1ca3714dc9807104173671f625, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733616855733 2024-12-08T00:14:17,812 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6c496075ee74fcc9e340228b3b1da4f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733616855492 2024-12-08T00:14:17,812 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 1191795e83b64d8ea14cf1fb8692291d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733616856088 2024-12-08T00:14:17,813 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c567277f2440eba58cbf472bc2ae50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733616855733 2024-12-08T00:14:17,813 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ba529cc32e7d408b9bb69cc5698d17ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733616856549 2024-12-08T00:14:17,814 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8968646c7a8841939998cbc5d3a8b590, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733616856088 2024-12-08T00:14:17,815 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4419cf09051b40d4b3027813c63b19cf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733616856549 2024-12-08T00:14:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:17,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T00:14:17,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:17,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:17,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:17,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:17,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:17,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/12d3cedc9ceb4fd8920a6d22daa2a6ce is 50, key is test_row_0/A:col10/1733616857148/Put/seqid=0 2024-12-08T00:14:17,847 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#B#compaction#21 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:17,847 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#A#compaction#22 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:17,848 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/8d26beab8adb434eb44f1aef27ddccc9 is 50, key is test_row_0/B:col10/1733616856557/Put/seqid=0 2024-12-08T00:14:17,848 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/90beaa7cede94a41885e1a76d55bac1e is 50, key is test_row_0/A:col10/1733616856557/Put/seqid=0 2024-12-08T00:14:17,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741858_1034 (size=12151) 2024-12-08T00:14:17,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/12d3cedc9ceb4fd8920a6d22daa2a6ce 2024-12-08T00:14:17,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:17,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:17,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T00:14:17,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741859_1035 (size=12391) 2024-12-08T00:14:17,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741860_1036 (size=12391) 2024-12-08T00:14:17,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/8d26beab8adb434eb44f1aef27ddccc9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d26beab8adb434eb44f1aef27ddccc9 2024-12-08T00:14:17,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/1e4609b0523d442d808d8e248c1d2201 is 50, key is test_row_0/B:col10/1733616857148/Put/seqid=0 2024-12-08T00:14:17,902 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/90beaa7cede94a41885e1a76d55bac1e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/90beaa7cede94a41885e1a76d55bac1e 2024-12-08T00:14:17,910 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/B of 9938c46f9ca7a3abf34703707edea068 into 8d26beab8adb434eb44f1aef27ddccc9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:17,910 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:17,910 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/B, priority=12, startTime=1733616857805; duration=0sec 2024-12-08T00:14:17,910 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:17,910 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:B 2024-12-08T00:14:17,911 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:17,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741861_1037 (size=12151) 2024-12-08T00:14:17,922 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:17,922 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/C is initiating minor compaction (all files) 2024-12-08T00:14:17,922 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/C in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:17,922 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7392da633af3454f9a7ee5e07b802f24, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/125d6532c359489c86dbef983f7c9191, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c630d29f70f24b9ca2cdf8fd1cf1143c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/96f829f08229456391a1970bb37a2091] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=47.1 K 2024-12-08T00:14:17,924 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 7392da633af3454f9a7ee5e07b802f24, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733616855492 2024-12-08T00:14:17,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616917915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,926 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 125d6532c359489c86dbef983f7c9191, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733616855733 2024-12-08T00:14:17,927 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c630d29f70f24b9ca2cdf8fd1cf1143c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733616856088 2024-12-08T00:14:17,928 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f829f08229456391a1970bb37a2091, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733616856549 2024-12-08T00:14:17,929 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/A of 9938c46f9ca7a3abf34703707edea068 into 90beaa7cede94a41885e1a76d55bac1e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:17,929 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:17,929 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/A, priority=12, startTime=1733616857805; duration=0sec 2024-12-08T00:14:17,929 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:17,929 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:A 2024-12-08T00:14:17,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616917925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616917931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:17,952 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#C#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:17,953 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/f05e3dfb3b754549b87066c6b9c0547d is 50, key is test_row_0/C:col10/1733616856557/Put/seqid=0 2024-12-08T00:14:17,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741862_1038 (size=12391) 2024-12-08T00:14:18,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616918031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616918051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616918051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616918241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616918261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616918261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/1e4609b0523d442d808d8e248c1d2201 2024-12-08T00:14:18,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/e4db68d6bc6c4f0d9f7a5123dbe32c64 is 50, key is test_row_0/C:col10/1733616857148/Put/seqid=0 2024-12-08T00:14:18,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:18,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:18,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741863_1039 (size=12151) 2024-12-08T00:14:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T00:14:18,389 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/f05e3dfb3b754549b87066c6b9c0547d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f05e3dfb3b754549b87066c6b9c0547d 2024-12-08T00:14:18,401 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/C of 9938c46f9ca7a3abf34703707edea068 into f05e3dfb3b754549b87066c6b9c0547d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:18,401 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:18,401 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/C, priority=12, startTime=1733616857806; duration=0sec 2024-12-08T00:14:18,402 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:18,402 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:C 2024-12-08T00:14:18,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:18,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:18,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616918551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616918571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616918571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:18,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:18,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:18,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616918730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,735 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 9938c46f9ca7a3abf34703707edea068 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/e4db68d6bc6c4f0d9f7a5123dbe32c64 2024-12-08T00:14:18,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616918751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:18,754 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 9938c46f9ca7a3abf34703707edea068 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:18,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/12d3cedc9ceb4fd8920a6d22daa2a6ce as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/12d3cedc9ceb4fd8920a6d22daa2a6ce 2024-12-08T00:14:18,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/12d3cedc9ceb4fd8920a6d22daa2a6ce, entries=150, sequenceid=152, filesize=11.9 K 2024-12-08T00:14:18,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/1e4609b0523d442d808d8e248c1d2201 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1e4609b0523d442d808d8e248c1d2201 2024-12-08T00:14:18,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1e4609b0523d442d808d8e248c1d2201, entries=150, sequenceid=152, filesize=11.9 K 2024-12-08T00:14:18,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/e4db68d6bc6c4f0d9f7a5123dbe32c64 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/e4db68d6bc6c4f0d9f7a5123dbe32c64 2024-12-08T00:14:18,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/e4db68d6bc6c4f0d9f7a5123dbe32c64, entries=150, sequenceid=152, filesize=11.9 K 2024-12-08T00:14:18,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 9938c46f9ca7a3abf34703707edea068 in 976ms, sequenceid=152, compaction requested=false 2024-12-08T00:14:18,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:18,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T00:14:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:18,802 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T00:14:18,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:18,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:18,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:18,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:18,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:18,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/3fbd23a40e25420c9fb8f26727db43e6 is 50, key is test_row_0/A:col10/1733616857917/Put/seqid=0 2024-12-08T00:14:18,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741864_1040 (size=12151) 2024-12-08T00:14:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:19,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:19,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616919104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616919113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616919118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616919221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616919221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616919231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,252 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/3fbd23a40e25420c9fb8f26727db43e6 2024-12-08T00:14:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/9513116425484938a8cb87e5d4145934 is 50, key is test_row_0/B:col10/1733616857917/Put/seqid=0 2024-12-08T00:14:19,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741865_1041 (size=12151) 2024-12-08T00:14:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T00:14:19,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616919431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616919431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616919441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,685 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/9513116425484938a8cb87e5d4145934 2024-12-08T00:14:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7cd1471a8c434ce9b6c4c3a383fad3a5 is 50, key is test_row_0/C:col10/1733616857917/Put/seqid=0 2024-12-08T00:14:19,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741866_1042 (size=12151) 2024-12-08T00:14:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616919741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616919741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616919751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7cd1471a8c434ce9b6c4c3a383fad3a5 2024-12-08T00:14:20,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/3fbd23a40e25420c9fb8f26727db43e6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3fbd23a40e25420c9fb8f26727db43e6 2024-12-08T00:14:20,132 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3fbd23a40e25420c9fb8f26727db43e6, entries=150, sequenceid=171, filesize=11.9 K 2024-12-08T00:14:20,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/9513116425484938a8cb87e5d4145934 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/9513116425484938a8cb87e5d4145934 2024-12-08T00:14:20,141 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/9513116425484938a8cb87e5d4145934, entries=150, sequenceid=171, filesize=11.9 K 2024-12-08T00:14:20,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7cd1471a8c434ce9b6c4c3a383fad3a5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7cd1471a8c434ce9b6c4c3a383fad3a5 2024-12-08T00:14:20,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7cd1471a8c434ce9b6c4c3a383fad3a5, entries=150, sequenceid=171, filesize=11.9 K 2024-12-08T00:14:20,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 9938c46f9ca7a3abf34703707edea068 in 1350ms, sequenceid=171, compaction requested=true 2024-12-08T00:14:20,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:20,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:20,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-08T00:14:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-08T00:14:20,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-08T00:14:20,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9130 sec 2024-12-08T00:14:20,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.9230 sec 2024-12-08T00:14:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:20,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T00:14:20,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:20,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:20,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:20,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:20,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:20,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:20,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/4148452665324730b0c85d3b0c0a21a7 is 50, key is test_row_0/A:col10/1733616860252/Put/seqid=0 2024-12-08T00:14:20,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741867_1043 (size=14541) 2024-12-08T00:14:20,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616920288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616920289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616920296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616920401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616920401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616920401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616920620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616920621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616920621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/4148452665324730b0c85d3b0c0a21a7 2024-12-08T00:14:20,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/e464b227259745d4b1548e8b9af7f35e is 50, key is test_row_0/B:col10/1733616860252/Put/seqid=0 2024-12-08T00:14:20,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741868_1044 (size=12151) 2024-12-08T00:14:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616920751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,754 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 9938c46f9ca7a3abf34703707edea068 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616920771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,773 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 9938c46f9ca7a3abf34703707edea068 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616920930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616920931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616920931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/e464b227259745d4b1548e8b9af7f35e 2024-12-08T00:14:21,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7d343654c25c4c4cacfd828491e8e579 is 50, key is test_row_0/C:col10/1733616860252/Put/seqid=0 2024-12-08T00:14:21,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741869_1045 (size=12151) 2024-12-08T00:14:21,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7d343654c25c4c4cacfd828491e8e579 2024-12-08T00:14:21,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/4148452665324730b0c85d3b0c0a21a7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4148452665324730b0c85d3b0c0a21a7 2024-12-08T00:14:21,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4148452665324730b0c85d3b0c0a21a7, entries=200, sequenceid=189, filesize=14.2 K 2024-12-08T00:14:21,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/e464b227259745d4b1548e8b9af7f35e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e464b227259745d4b1548e8b9af7f35e 2024-12-08T00:14:21,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e464b227259745d4b1548e8b9af7f35e, entries=150, sequenceid=189, filesize=11.9 K 2024-12-08T00:14:21,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/7d343654c25c4c4cacfd828491e8e579 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7d343654c25c4c4cacfd828491e8e579 2024-12-08T00:14:21,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7d343654c25c4c4cacfd828491e8e579, entries=150, sequenceid=189, filesize=11.9 K 2024-12-08T00:14:21,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9938c46f9ca7a3abf34703707edea068 in 945ms, sequenceid=189, compaction requested=true 2024-12-08T00:14:21,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:21,199 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:21,199 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:21,201 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51234 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:21,201 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:21,201 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/A is initiating minor compaction (all files) 2024-12-08T00:14:21,201 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/B is initiating minor compaction (all files) 2024-12-08T00:14:21,202 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/A in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:21,202 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/90beaa7cede94a41885e1a76d55bac1e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/12d3cedc9ceb4fd8920a6d22daa2a6ce, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3fbd23a40e25420c9fb8f26727db43e6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4148452665324730b0c85d3b0c0a21a7] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=50.0 K 2024-12-08T00:14:21,202 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/B in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:21,202 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d26beab8adb434eb44f1aef27ddccc9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1e4609b0523d442d808d8e248c1d2201, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/9513116425484938a8cb87e5d4145934, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e464b227259745d4b1548e8b9af7f35e] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=47.7 K 2024-12-08T00:14:21,202 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90beaa7cede94a41885e1a76d55bac1e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733616856549 2024-12-08T00:14:21,203 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d26beab8adb434eb44f1aef27ddccc9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733616856549 2024-12-08T00:14:21,203 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4609b0523d442d808d8e248c1d2201, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733616857148 2024-12-08T00:14:21,203 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d3cedc9ceb4fd8920a6d22daa2a6ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733616857148 2024-12-08T00:14:21,204 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fbd23a40e25420c9fb8f26727db43e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733616857893 2024-12-08T00:14:21,204 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 9513116425484938a8cb87e5d4145934, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733616857893 2024-12-08T00:14:21,204 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4148452665324730b0c85d3b0c0a21a7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733616859093 2024-12-08T00:14:21,204 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e464b227259745d4b1548e8b9af7f35e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733616859093 2024-12-08T00:14:21,243 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#B#compaction#33 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:21,244 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/8d12b255daca457daf2077e54c53aa08 is 50, key is test_row_0/B:col10/1733616860252/Put/seqid=0 2024-12-08T00:14:21,245 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#A#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:21,246 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/4831446ce420462b913f416032bf9a37 is 50, key is test_row_0/A:col10/1733616860252/Put/seqid=0 2024-12-08T00:14:21,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741870_1046 (size=12527) 2024-12-08T00:14:21,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741871_1047 (size=12527) 2024-12-08T00:14:21,299 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/8d12b255daca457daf2077e54c53aa08 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d12b255daca457daf2077e54c53aa08 2024-12-08T00:14:21,307 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/4831446ce420462b913f416032bf9a37 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4831446ce420462b913f416032bf9a37 2024-12-08T00:14:21,312 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/B of 9938c46f9ca7a3abf34703707edea068 into 8d12b255daca457daf2077e54c53aa08(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:21,312 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:21,312 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/B, priority=12, startTime=1733616861199; duration=0sec 2024-12-08T00:14:21,313 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:21,313 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:B 2024-12-08T00:14:21,313 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:21,318 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/A of 9938c46f9ca7a3abf34703707edea068 into 4831446ce420462b913f416032bf9a37(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:21,318 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:21,318 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/A, priority=12, startTime=1733616861199; duration=0sec 2024-12-08T00:14:21,318 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:21,318 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:A 2024-12-08T00:14:21,319 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:21,319 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/C is initiating minor compaction (all files) 2024-12-08T00:14:21,319 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/C in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:21,319 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f05e3dfb3b754549b87066c6b9c0547d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/e4db68d6bc6c4f0d9f7a5123dbe32c64, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7cd1471a8c434ce9b6c4c3a383fad3a5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7d343654c25c4c4cacfd828491e8e579] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=47.7 K 2024-12-08T00:14:21,320 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting f05e3dfb3b754549b87066c6b9c0547d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733616856549 2024-12-08T00:14:21,321 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e4db68d6bc6c4f0d9f7a5123dbe32c64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733616857148 2024-12-08T00:14:21,322 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cd1471a8c434ce9b6c4c3a383fad3a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733616857893 2024-12-08T00:14:21,322 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d343654c25c4c4cacfd828491e8e579, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733616859093 2024-12-08T00:14:21,339 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#C#compaction#35 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:21,340 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/c1104ffef3a94f4a99c3c8efda4ca87a is 50, key is test_row_0/C:col10/1733616860252/Put/seqid=0 2024-12-08T00:14:21,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741872_1048 (size=12527) 2024-12-08T00:14:21,360 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/c1104ffef3a94f4a99c3c8efda4ca87a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c1104ffef3a94f4a99c3c8efda4ca87a 2024-12-08T00:14:21,373 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/C of 9938c46f9ca7a3abf34703707edea068 into c1104ffef3a94f4a99c3c8efda4ca87a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:21,373 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:21,373 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/C, priority=12, startTime=1733616861199; duration=0sec 2024-12-08T00:14:21,373 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:21,373 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:C 2024-12-08T00:14:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T00:14:21,412 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:21,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-08T00:14:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T00:14:21,415 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:21,417 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:21,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:21,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T00:14:21,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:21,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:21,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:21,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:21,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:21,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:21,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/b2770038c88c450b9dd4c7257efd3f67 is 50, key is test_row_0/A:col10/1733616860287/Put/seqid=0 2024-12-08T00:14:21,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741873_1049 (size=12151) 2024-12-08T00:14:21,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/b2770038c88c450b9dd4c7257efd3f67 2024-12-08T00:14:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616921488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616921491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616921492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/5dfbe24f4a5b45f7b5447ef9ed6ce132 is 50, key is test_row_0/B:col10/1733616860287/Put/seqid=0 2024-12-08T00:14:21,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741874_1050 (size=12151) 2024-12-08T00:14:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T00:14:21,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T00:14:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:21,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:21,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:21,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616921601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616921601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616921602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T00:14:21,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:21,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T00:14:21,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:21,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616921810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616921811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616921811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:21,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T00:14:21,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:21,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:21,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:21,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:21,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:21,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/5dfbe24f4a5b45f7b5447ef9ed6ce132 2024-12-08T00:14:21,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/c450910dd1a04729ba306c4ca072fdd1 is 50, key is test_row_0/C:col10/1733616860287/Put/seqid=0 2024-12-08T00:14:21,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741875_1051 (size=12151) 2024-12-08T00:14:22,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T00:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:22,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T00:14:22,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616922120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616922121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616922121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T00:14:22,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:22,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:22,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:22,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:22,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:22,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/c450910dd1a04729ba306c4ca072fdd1 2024-12-08T00:14:22,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T00:14:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:22,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:22,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:22,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/b2770038c88c450b9dd4c7257efd3f67 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b2770038c88c450b9dd4c7257efd3f67 2024-12-08T00:14:22,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b2770038c88c450b9dd4c7257efd3f67, entries=150, sequenceid=209, filesize=11.9 K 2024-12-08T00:14:22,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/5dfbe24f4a5b45f7b5447ef9ed6ce132 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/5dfbe24f4a5b45f7b5447ef9ed6ce132 2024-12-08T00:14:22,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/5dfbe24f4a5b45f7b5447ef9ed6ce132, entries=150, sequenceid=209, filesize=11.9 K 2024-12-08T00:14:22,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/c450910dd1a04729ba306c4ca072fdd1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c450910dd1a04729ba306c4ca072fdd1 2024-12-08T00:14:22,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c450910dd1a04729ba306c4ca072fdd1, entries=150, sequenceid=209, filesize=11.9 K 2024-12-08T00:14:22,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9938c46f9ca7a3abf34703707edea068 in 919ms, sequenceid=209, compaction requested=false 2024-12-08T00:14:22,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:22,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T00:14:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:22,494 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T00:14:22,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:22,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:22,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:22,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:22,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:22,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:22,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8a286f11bacb4a79b85947a0330cb12d is 50, key is test_row_0/A:col10/1733616861479/Put/seqid=0 2024-12-08T00:14:22,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741876_1052 (size=12151) 2024-12-08T00:14:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T00:14:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:22,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616922652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616922654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616922655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616922761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616922761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616922761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,919 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8a286f11bacb4a79b85947a0330cb12d 2024-12-08T00:14:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/ee132481fbd0443d9990ef8f98da9c8a is 50, key is test_row_0/B:col10/1733616861479/Put/seqid=0 2024-12-08T00:14:22,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741877_1053 (size=12151) 2024-12-08T00:14:22,932 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/ee132481fbd0443d9990ef8f98da9c8a 2024-12-08T00:14:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/6267782f7d9649ee8c435f1b12959fdc is 50, key is test_row_0/C:col10/1733616861479/Put/seqid=0 2024-12-08T00:14:22,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741878_1054 (size=12151) 2024-12-08T00:14:22,960 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/6267782f7d9649ee8c435f1b12959fdc 2024-12-08T00:14:22,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616922971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616922972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8a286f11bacb4a79b85947a0330cb12d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8a286f11bacb4a79b85947a0330cb12d 2024-12-08T00:14:22,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616922972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:22,986 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8a286f11bacb4a79b85947a0330cb12d, entries=150, sequenceid=224, filesize=11.9 K 2024-12-08T00:14:22,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/ee132481fbd0443d9990ef8f98da9c8a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ee132481fbd0443d9990ef8f98da9c8a 2024-12-08T00:14:23,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ee132481fbd0443d9990ef8f98da9c8a, entries=150, sequenceid=224, filesize=11.9 K 2024-12-08T00:14:23,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/6267782f7d9649ee8c435f1b12959fdc as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/6267782f7d9649ee8c435f1b12959fdc 2024-12-08T00:14:23,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/6267782f7d9649ee8c435f1b12959fdc, entries=150, sequenceid=224, filesize=11.9 K 2024-12-08T00:14:23,009 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9938c46f9ca7a3abf34703707edea068 in 515ms, sequenceid=224, compaction requested=true 2024-12-08T00:14:23,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:23,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:23,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-08T00:14:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-08T00:14:23,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T00:14:23,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5940 sec 2024-12-08T00:14:23,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6030 sec 2024-12-08T00:14:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:23,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T00:14:23,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:23,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:23,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:23,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:23,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:23,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:23,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/d66f6bcb7f3c48009e81cba53ed9212b is 50, key is test_row_0/A:col10/1733616862653/Put/seqid=0 2024-12-08T00:14:23,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741879_1055 (size=14541) 2024-12-08T00:14:23,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616923299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:23,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:23,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616923300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:23,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616923302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616923411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616923411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:23,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616923411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T00:14:23,561 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:23,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-08T00:14:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T00:14:23,564 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:23,565 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:23,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:23,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616923621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:23,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616923621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616923621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T00:14:23,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/d66f6bcb7f3c48009e81cba53ed9212b 2024-12-08T00:14:23,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/08132d8bd2cd4adba0dcc8527adc275a is 50, key is test_row_0/B:col10/1733616862653/Put/seqid=0 2024-12-08T00:14:23,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T00:14:23,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:23,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:23,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:23,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:23,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:23,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741880_1056 (size=12151) 2024-12-08T00:14:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:23,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/08132d8bd2cd4adba0dcc8527adc275a 2024-12-08T00:14:23,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/4d3667ddf0624aabb5369f9f97bc5c05 is 50, key is test_row_0/C:col10/1733616862653/Put/seqid=0 2024-12-08T00:14:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741881_1057 (size=12151) 2024-12-08T00:14:23,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/4d3667ddf0624aabb5369f9f97bc5c05 2024-12-08T00:14:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/d66f6bcb7f3c48009e81cba53ed9212b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d66f6bcb7f3c48009e81cba53ed9212b 2024-12-08T00:14:23,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d66f6bcb7f3c48009e81cba53ed9212b, entries=200, sequenceid=243, filesize=14.2 K 2024-12-08T00:14:23,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/08132d8bd2cd4adba0dcc8527adc275a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/08132d8bd2cd4adba0dcc8527adc275a 2024-12-08T00:14:23,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/08132d8bd2cd4adba0dcc8527adc275a, entries=150, sequenceid=243, filesize=11.9 K 2024-12-08T00:14:23,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/4d3667ddf0624aabb5369f9f97bc5c05 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/4d3667ddf0624aabb5369f9f97bc5c05 2024-12-08T00:14:23,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/4d3667ddf0624aabb5369f9f97bc5c05, entries=150, sequenceid=243, filesize=11.9 K 2024-12-08T00:14:23,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9938c46f9ca7a3abf34703707edea068 in 509ms, sequenceid=243, compaction requested=true 2024-12-08T00:14:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:23,792 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:23,793 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:23,795 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:23,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:23,795 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51370 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:23,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:23,795 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/B is initiating minor compaction (all files) 2024-12-08T00:14:23,795 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/A is initiating minor compaction (all files) 2024-12-08T00:14:23,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:23,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:23,795 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/B in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:23,795 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/A in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:23,795 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d12b255daca457daf2077e54c53aa08, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/5dfbe24f4a5b45f7b5447ef9ed6ce132, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ee132481fbd0443d9990ef8f98da9c8a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/08132d8bd2cd4adba0dcc8527adc275a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=47.8 K 2024-12-08T00:14:23,795 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4831446ce420462b913f416032bf9a37, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b2770038c88c450b9dd4c7257efd3f67, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8a286f11bacb4a79b85947a0330cb12d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d66f6bcb7f3c48009e81cba53ed9212b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=50.2 K 2024-12-08T00:14:23,797 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4831446ce420462b913f416032bf9a37, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733616859093 2024-12-08T00:14:23,797 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d12b255daca457daf2077e54c53aa08, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733616859093 2024-12-08T00:14:23,798 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2770038c88c450b9dd4c7257efd3f67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733616860287 2024-12-08T00:14:23,798 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dfbe24f4a5b45f7b5447ef9ed6ce132, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733616860287 2024-12-08T00:14:23,798 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a286f11bacb4a79b85947a0330cb12d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733616861479 2024-12-08T00:14:23,798 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ee132481fbd0443d9990ef8f98da9c8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733616861479 2024-12-08T00:14:23,799 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting d66f6bcb7f3c48009e81cba53ed9212b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733616862633 2024-12-08T00:14:23,799 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 08132d8bd2cd4adba0dcc8527adc275a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733616862653 2024-12-08T00:14:23,823 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:23,824 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/3426588e9fb24e2fab33df8451de69aa is 50, key is test_row_0/A:col10/1733616862653/Put/seqid=0 2024-12-08T00:14:23,827 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:23,828 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/205e031b99f247728294cdb79bdc5d79 is 50, key is test_row_0/B:col10/1733616862653/Put/seqid=0 2024-12-08T00:14:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741882_1058 (size=12663) 2024-12-08T00:14:23,843 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/3426588e9fb24e2fab33df8451de69aa as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3426588e9fb24e2fab33df8451de69aa 2024-12-08T00:14:23,851 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/A of 9938c46f9ca7a3abf34703707edea068 into 3426588e9fb24e2fab33df8451de69aa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:23,851 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:23,851 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/A, priority=12, startTime=1733616863792; duration=0sec 2024-12-08T00:14:23,851 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:23,851 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:A 2024-12-08T00:14:23,851 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:23,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741883_1059 (size=12663) 2024-12-08T00:14:23,859 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:23,859 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/C is initiating minor compaction (all files) 2024-12-08T00:14:23,859 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/C in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:23,859 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c1104ffef3a94f4a99c3c8efda4ca87a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c450910dd1a04729ba306c4ca072fdd1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/6267782f7d9649ee8c435f1b12959fdc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/4d3667ddf0624aabb5369f9f97bc5c05] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=47.8 K 2024-12-08T00:14:23,860 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1104ffef3a94f4a99c3c8efda4ca87a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733616859093 2024-12-08T00:14:23,862 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting c450910dd1a04729ba306c4ca072fdd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733616860287 2024-12-08T00:14:23,865 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6267782f7d9649ee8c435f1b12959fdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733616861479 2024-12-08T00:14:23,865 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/205e031b99f247728294cdb79bdc5d79 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/205e031b99f247728294cdb79bdc5d79 2024-12-08T00:14:23,866 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3667ddf0624aabb5369f9f97bc5c05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733616862653 2024-12-08T00:14:23,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T00:14:23,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:23,877 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/B of 9938c46f9ca7a3abf34703707edea068 into 205e031b99f247728294cdb79bdc5d79(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:23,877 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:23,877 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:14:23,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:23,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:23,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:23,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:23,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:23,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:23,878 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/B, priority=12, startTime=1733616863792; duration=0sec 2024-12-08T00:14:23,878 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:23,879 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:B 2024-12-08T00:14:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T00:14:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8e9b1c2bc0b248d1abbfd7852ddcd130 is 50, key is test_row_0/A:col10/1733616863294/Put/seqid=0 2024-12-08T00:14:23,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741884_1060 (size=12151) 2024-12-08T00:14:23,895 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:23,895 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8e9b1c2bc0b248d1abbfd7852ddcd130 2024-12-08T00:14:23,896 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/f137d857fd7a4695b4529dfdaa04b813 is 50, key is test_row_0/C:col10/1733616862653/Put/seqid=0 2024-12-08T00:14:23,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741885_1061 (size=12663) 2024-12-08T00:14:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/71ced269e0ca4b80ad979ab7df38e41f is 50, key is test_row_0/B:col10/1733616863294/Put/seqid=0 2024-12-08T00:14:23,918 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/f137d857fd7a4695b4529dfdaa04b813 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f137d857fd7a4695b4529dfdaa04b813 2024-12-08T00:14:23,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741886_1062 (size=12151) 2024-12-08T00:14:23,929 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/C of 9938c46f9ca7a3abf34703707edea068 into f137d857fd7a4695b4529dfdaa04b813(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:23,929 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:23,930 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/C, priority=12, startTime=1733616863795; duration=0sec 2024-12-08T00:14:23,930 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:23,930 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:C 2024-12-08T00:14:23,930 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/71ced269e0ca4b80ad979ab7df38e41f 2024-12-08T00:14:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:23,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/d2a965ac80b546ccbd3d013731718c11 is 50, key is test_row_0/C:col10/1733616863294/Put/seqid=0 2024-12-08T00:14:23,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741887_1063 (size=12151) 2024-12-08T00:14:24,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=211, cellSize=9656, totalHeapSize=22184, min timestamp=1733616863937, max timestamp=1733616863987Num uniques 859; 2024-12-08T00:14:24,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616863937, max timestamp=1733616863987Num uniques 859; 2024-12-08T00:14:24,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9656, suffix heap size=117064,new segment heap size=22184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=211 2024-12-08T00:14:24,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:14:24,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616924014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616924021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616924026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616924131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616924131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616924131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T00:14:24,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616924341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616924341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616924341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,371 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/d2a965ac80b546ccbd3d013731718c11 2024-12-08T00:14:24,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8e9b1c2bc0b248d1abbfd7852ddcd130 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8e9b1c2bc0b248d1abbfd7852ddcd130 2024-12-08T00:14:24,383 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8e9b1c2bc0b248d1abbfd7852ddcd130, entries=150, sequenceid=258, filesize=11.9 K 2024-12-08T00:14:24,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/71ced269e0ca4b80ad979ab7df38e41f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/71ced269e0ca4b80ad979ab7df38e41f 2024-12-08T00:14:24,390 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/71ced269e0ca4b80ad979ab7df38e41f, entries=150, sequenceid=258, filesize=11.9 K 2024-12-08T00:14:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/d2a965ac80b546ccbd3d013731718c11 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/d2a965ac80b546ccbd3d013731718c11 2024-12-08T00:14:24,401 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/d2a965ac80b546ccbd3d013731718c11, entries=150, sequenceid=258, filesize=11.9 K 2024-12-08T00:14:24,403 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=109.67 KB/112302 for 9938c46f9ca7a3abf34703707edea068 in 525ms, sequenceid=258, compaction requested=false 2024-12-08T00:14:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-08T00:14:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-08T00:14:24,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-08T00:14:24,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 839 msec 2024-12-08T00:14:24,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 847 msec 2024-12-08T00:14:24,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=123.09 KB heapSize=353.83 KB 2024-12-08T00:14:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:24,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/af22e6cb1c474d70a1022aaa6758b6b6 is 50, key is test_row_0/A:col10/1733616864010/Put/seqid=0 2024-12-08T00:14:24,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616924667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741888_1064 (size=14741) 2024-12-08T00:14:24,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.63 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/af22e6cb1c474d70a1022aaa6758b6b6 2024-12-08T00:14:24,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616924670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616924671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/d0777084916642f3b7edb4c98fad72d0 is 50, key is test_row_0/B:col10/1733616864010/Put/seqid=0 2024-12-08T00:14:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T00:14:24,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741889_1065 (size=12301) 2024-12-08T00:14:24,703 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:24,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-08T00:14:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T00:14:24,705 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:24,706 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:24,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:24,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39024 deadline: 1733616924761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,762 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 9938c46f9ca7a3abf34703707edea068 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616924780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616924781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616924791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T00:14:24,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39018 deadline: 1733616924811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,813 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 9938c46f9ca7a3abf34703707edea068 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T00:14:24,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:24,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:24,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:24,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:24,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:24,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616924990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616924990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616925001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T00:14:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:25,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T00:14:25,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/d0777084916642f3b7edb4c98fad72d0 2024-12-08T00:14:25,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/ea7158b411e143a4adb2dd619a79cc17 is 50, key is test_row_0/C:col10/1733616864010/Put/seqid=0 2024-12-08T00:14:25,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741890_1066 (size=12301) 2024-12-08T00:14:25,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/ea7158b411e143a4adb2dd619a79cc17 2024-12-08T00:14:25,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/af22e6cb1c474d70a1022aaa6758b6b6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/af22e6cb1c474d70a1022aaa6758b6b6 2024-12-08T00:14:25,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/af22e6cb1c474d70a1022aaa6758b6b6, entries=200, sequenceid=290, filesize=14.4 K 2024-12-08T00:14:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/d0777084916642f3b7edb4c98fad72d0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/d0777084916642f3b7edb4c98fad72d0 2024-12-08T00:14:25,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/d0777084916642f3b7edb4c98fad72d0, entries=150, sequenceid=290, filesize=12.0 K 2024-12-08T00:14:25,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/ea7158b411e143a4adb2dd619a79cc17 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/ea7158b411e143a4adb2dd619a79cc17 2024-12-08T00:14:25,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T00:14:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/ea7158b411e143a4adb2dd619a79cc17, entries=150, sequenceid=290, filesize=12.0 K 2024-12-08T00:14:25,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.80 KB/132912, heapSize ~373.70 KB/382672, currentSize=46.96 KB/48090 for 9938c46f9ca7a3abf34703707edea068 in 524ms, sequenceid=290, compaction requested=true 2024-12-08T00:14:25,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:25,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:25,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:25,178 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:25,178 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:25,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:25,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:25,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:25,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:25,183 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:25,183 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/B is initiating minor compaction (all files) 2024-12-08T00:14:25,183 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/B in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,183 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/205e031b99f247728294cdb79bdc5d79, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/71ced269e0ca4b80ad979ab7df38e41f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/d0777084916642f3b7edb4c98fad72d0] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=36.2 K 2024-12-08T00:14:25,184 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:25,184 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/A is initiating minor compaction (all files) 2024-12-08T00:14:25,184 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/A in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,184 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3426588e9fb24e2fab33df8451de69aa, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8e9b1c2bc0b248d1abbfd7852ddcd130, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/af22e6cb1c474d70a1022aaa6758b6b6] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=38.6 K 2024-12-08T00:14:25,186 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3426588e9fb24e2fab33df8451de69aa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733616862653 2024-12-08T00:14:25,186 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 205e031b99f247728294cdb79bdc5d79, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733616862653 2024-12-08T00:14:25,187 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9b1c2bc0b248d1abbfd7852ddcd130, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733616863294 2024-12-08T00:14:25,187 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ced269e0ca4b80ad979ab7df38e41f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733616863294 2024-12-08T00:14:25,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting af22e6cb1c474d70a1022aaa6758b6b6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733616864010 2024-12-08T00:14:25,189 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d0777084916642f3b7edb4c98fad72d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733616864010 2024-12-08T00:14:25,211 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:25,212 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/6705e62516e24615bd8e30f7b9f708d6 is 50, key is test_row_0/B:col10/1733616864010/Put/seqid=0 2024-12-08T00:14:25,217 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#A#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:25,218 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/c0f0bd6d9a854a7985cc25e5a9d6b9d5 is 50, key is test_row_0/A:col10/1733616864010/Put/seqid=0 2024-12-08T00:14:25,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741891_1067 (size=12915) 2024-12-08T00:14:25,253 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/6705e62516e24615bd8e30f7b9f708d6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/6705e62516e24615bd8e30f7b9f708d6 2024-12-08T00:14:25,262 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/B of 9938c46f9ca7a3abf34703707edea068 into 6705e62516e24615bd8e30f7b9f708d6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:25,262 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:25,262 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/B, priority=13, startTime=1733616865178; duration=0sec 2024-12-08T00:14:25,262 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:25,262 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:B 2024-12-08T00:14:25,262 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:25,264 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:25,264 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/C is initiating minor compaction (all files) 2024-12-08T00:14:25,265 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/C in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,265 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f137d857fd7a4695b4529dfdaa04b813, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/d2a965ac80b546ccbd3d013731718c11, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/ea7158b411e143a4adb2dd619a79cc17] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=36.2 K 2024-12-08T00:14:25,265 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting f137d857fd7a4695b4529dfdaa04b813, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733616862653 2024-12-08T00:14:25,266 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a965ac80b546ccbd3d013731718c11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733616863294 2024-12-08T00:14:25,267 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ea7158b411e143a4adb2dd619a79cc17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733616864010 2024-12-08T00:14:25,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741892_1068 (size=12915) 2024-12-08T00:14:25,280 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/c0f0bd6d9a854a7985cc25e5a9d6b9d5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/c0f0bd6d9a854a7985cc25e5a9d6b9d5 2024-12-08T00:14:25,283 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:25,283 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/138dac1b11cb46d888add886a922ef46 is 50, key is test_row_0/C:col10/1733616864010/Put/seqid=0 2024-12-08T00:14:25,299 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/A of 9938c46f9ca7a3abf34703707edea068 into c0f0bd6d9a854a7985cc25e5a9d6b9d5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:25,299 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:25,299 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/A, priority=13, startTime=1733616865178; duration=0sec 2024-12-08T00:14:25,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741893_1069 (size=12915) 2024-12-08T00:14:25,300 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:25,300 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:A 2024-12-08T00:14:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:25,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T00:14:25,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:25,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:25,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:25,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:25,308 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/138dac1b11cb46d888add886a922ef46 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/138dac1b11cb46d888add886a922ef46 2024-12-08T00:14:25,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:25,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:25,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/e830a0d241a64777acf2bd4a78ecf9f7 is 50, key is test_row_0/A:col10/1733616865306/Put/seqid=0 2024-12-08T00:14:25,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T00:14:25,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:25,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,321 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/C of 9938c46f9ca7a3abf34703707edea068 into 138dac1b11cb46d888add886a922ef46(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:25,321 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:25,321 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/C, priority=13, startTime=1733616865179; duration=0sec 2024-12-08T00:14:25,322 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:25,322 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:C 2024-12-08T00:14:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741894_1070 (size=14741) 2024-12-08T00:14:25,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/e830a0d241a64777acf2bd4a78ecf9f7 2024-12-08T00:14:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T00:14:25,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/388e679f65524a81a291f6e360d71625 is 50, key is test_row_0/B:col10/1733616865306/Put/seqid=0 2024-12-08T00:14:25,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741895_1071 (size=12301) 2024-12-08T00:14:25,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616925349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616925352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616925354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616925461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616925462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616925462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T00:14:25,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:25,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T00:14:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616925671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616925671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616925671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:25,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/388e679f65524a81a291f6e360d71625 2024-12-08T00:14:25,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/314df25b45234d4f946bd5d85d774b57 is 50, key is test_row_0/C:col10/1733616865306/Put/seqid=0 2024-12-08T00:14:25,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741896_1072 (size=12301) 2024-12-08T00:14:25,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/314df25b45234d4f946bd5d85d774b57 2024-12-08T00:14:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/e830a0d241a64777acf2bd4a78ecf9f7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e830a0d241a64777acf2bd4a78ecf9f7 2024-12-08T00:14:25,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T00:14:25,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:25,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:25,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e830a0d241a64777acf2bd4a78ecf9f7, entries=200, sequenceid=304, filesize=14.4 K 2024-12-08T00:14:25,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/388e679f65524a81a291f6e360d71625 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/388e679f65524a81a291f6e360d71625 2024-12-08T00:14:25,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/388e679f65524a81a291f6e360d71625, entries=150, sequenceid=304, filesize=12.0 K 2024-12-08T00:14:25,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/314df25b45234d4f946bd5d85d774b57 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/314df25b45234d4f946bd5d85d774b57 2024-12-08T00:14:25,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/314df25b45234d4f946bd5d85d774b57, entries=150, sequenceid=304, filesize=12.0 K 2024-12-08T00:14:25,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9938c46f9ca7a3abf34703707edea068 in 519ms, sequenceid=304, compaction requested=false 2024-12-08T00:14:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T00:14:25,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T00:14:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:25,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:14:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8c54ca65a0554832b5fac54a2a162e1e is 50, key is test_row_0/A:col10/1733616865345/Put/seqid=0 2024-12-08T00:14:25,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741897_1073 (size=12301) 2024-12-08T00:14:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:25,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:26,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616925998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616925998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616925999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616926112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616926112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616926112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616926331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616926331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616926331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,361 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8c54ca65a0554832b5fac54a2a162e1e 2024-12-08T00:14:26,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/38281a9b47bb4adb908bb6d2b56c71e7 is 50, key is test_row_0/B:col10/1733616865345/Put/seqid=0 2024-12-08T00:14:26,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741898_1074 (size=12301) 2024-12-08T00:14:26,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616926641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616926641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616926641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:26,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/38281a9b47bb4adb908bb6d2b56c71e7 2024-12-08T00:14:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/a1aa6ccf67194add9092a437d48a4977 is 50, key is test_row_0/C:col10/1733616865345/Put/seqid=0 2024-12-08T00:14:26,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741899_1075 (size=12301) 2024-12-08T00:14:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T00:14:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616927150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:27,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616927150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:27,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616927161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:27,212 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/a1aa6ccf67194add9092a437d48a4977 2024-12-08T00:14:27,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/8c54ca65a0554832b5fac54a2a162e1e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8c54ca65a0554832b5fac54a2a162e1e 2024-12-08T00:14:27,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8c54ca65a0554832b5fac54a2a162e1e, entries=150, sequenceid=325, filesize=12.0 K 2024-12-08T00:14:27,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/38281a9b47bb4adb908bb6d2b56c71e7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/38281a9b47bb4adb908bb6d2b56c71e7 2024-12-08T00:14:27,230 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/38281a9b47bb4adb908bb6d2b56c71e7, entries=150, sequenceid=325, filesize=12.0 K 2024-12-08T00:14:27,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/a1aa6ccf67194add9092a437d48a4977 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a1aa6ccf67194add9092a437d48a4977 2024-12-08T00:14:27,237 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a1aa6ccf67194add9092a437d48a4977, entries=150, sequenceid=325, filesize=12.0 K 2024-12-08T00:14:27,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9938c46f9ca7a3abf34703707edea068 in 1291ms, sequenceid=325, compaction requested=true 2024-12-08T00:14:27,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:27,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:27,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-08T00:14:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-08T00:14:27,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-08T00:14:27,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5330 sec 2024-12-08T00:14:27,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.5380 sec 2024-12-08T00:14:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:28,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:14:28,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:28,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:28,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:28,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/aa526fcf7c264a98876374d964539a7c is 50, key is test_row_0/A:col10/1733616865988/Put/seqid=0 2024-12-08T00:14:28,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741900_1076 (size=14741) 2024-12-08T00:14:28,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616928199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616928199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616928200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616928310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616928310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616928310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616928521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616928521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616928521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/aa526fcf7c264a98876374d964539a7c 2024-12-08T00:14:28,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/3cffe53dd3b24c55b9a0a0034a961919 is 50, key is test_row_0/B:col10/1733616865988/Put/seqid=0 2024-12-08T00:14:28,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741901_1077 (size=12301) 2024-12-08T00:14:28,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616928831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616928831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616928831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T00:14:28,871 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:28,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-08T00:14:28,874 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T00:14:28,875 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:28,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T00:14:29,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/3cffe53dd3b24c55b9a0a0034a961919 2024-12-08T00:14:29,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/0404c929c6cc4b108acf9291306046de is 50, key is test_row_0/C:col10/1733616865988/Put/seqid=0 2024-12-08T00:14:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741902_1078 (size=12301) 2024-12-08T00:14:29,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T00:14:29,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:29,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T00:14:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T00:14:29,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T00:14:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:29,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616929340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:29,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616929340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:29,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616929340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:29,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/0404c929c6cc4b108acf9291306046de 2024-12-08T00:14:29,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/aa526fcf7c264a98876374d964539a7c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/aa526fcf7c264a98876374d964539a7c 2024-12-08T00:14:29,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/aa526fcf7c264a98876374d964539a7c, entries=200, sequenceid=338, filesize=14.4 K 2024-12-08T00:14:29,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/3cffe53dd3b24c55b9a0a0034a961919 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/3cffe53dd3b24c55b9a0a0034a961919 2024-12-08T00:14:29,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/3cffe53dd3b24c55b9a0a0034a961919, entries=150, sequenceid=338, filesize=12.0 K 2024-12-08T00:14:29,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/0404c929c6cc4b108acf9291306046de as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/0404c929c6cc4b108acf9291306046de 2024-12-08T00:14:29,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/0404c929c6cc4b108acf9291306046de, entries=150, sequenceid=338, filesize=12.0 K 2024-12-08T00:14:29,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 9938c46f9ca7a3abf34703707edea068 in 1285ms, sequenceid=338, compaction requested=true 2024-12-08T00:14:29,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:29,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:29,447 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:29,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:29,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:29,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:29,448 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:29,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:29,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:29,449 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:29,449 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:29,449 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/A is initiating minor compaction (all files) 2024-12-08T00:14:29,449 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/B is initiating minor compaction (all files) 2024-12-08T00:14:29,449 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/B in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,449 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/A in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,450 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/6705e62516e24615bd8e30f7b9f708d6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/388e679f65524a81a291f6e360d71625, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/38281a9b47bb4adb908bb6d2b56c71e7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/3cffe53dd3b24c55b9a0a0034a961919] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=48.7 K 2024-12-08T00:14:29,450 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/c0f0bd6d9a854a7985cc25e5a9d6b9d5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e830a0d241a64777acf2bd4a78ecf9f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8c54ca65a0554832b5fac54a2a162e1e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/aa526fcf7c264a98876374d964539a7c] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=53.4 K 2024-12-08T00:14:29,450 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6705e62516e24615bd8e30f7b9f708d6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733616864010 2024-12-08T00:14:29,450 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0f0bd6d9a854a7985cc25e5a9d6b9d5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733616864010 2024-12-08T00:14:29,450 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 388e679f65524a81a291f6e360d71625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733616864663 2024-12-08T00:14:29,450 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting e830a0d241a64777acf2bd4a78ecf9f7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733616864663 2024-12-08T00:14:29,451 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 38281a9b47bb4adb908bb6d2b56c71e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733616865345 2024-12-08T00:14:29,451 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c54ca65a0554832b5fac54a2a162e1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733616865345 2024-12-08T00:14:29,451 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa526fcf7c264a98876374d964539a7c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733616865988 2024-12-08T00:14:29,451 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cffe53dd3b24c55b9a0a0034a961919, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733616865988 2024-12-08T00:14:29,462 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:29,463 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/945e81b728c84d83ab67cae7fea2278b is 50, key is test_row_0/B:col10/1733616865988/Put/seqid=0 2024-12-08T00:14:29,465 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:29,466 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/85b8ccd25a64458c812cea2f2a186f4c is 50, key is test_row_0/A:col10/1733616865988/Put/seqid=0 2024-12-08T00:14:29,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741903_1079 (size=13051) 2024-12-08T00:14:29,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741904_1080 (size=13051) 2024-12-08T00:14:29,478 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/945e81b728c84d83ab67cae7fea2278b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/945e81b728c84d83ab67cae7fea2278b 2024-12-08T00:14:29,485 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/85b8ccd25a64458c812cea2f2a186f4c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/85b8ccd25a64458c812cea2f2a186f4c 2024-12-08T00:14:29,487 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/B of 9938c46f9ca7a3abf34703707edea068 into 945e81b728c84d83ab67cae7fea2278b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:29,487 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:29,487 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/B, priority=12, startTime=1733616869447; duration=0sec 2024-12-08T00:14:29,488 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:29,488 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:B 2024-12-08T00:14:29,488 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:14:29,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T00:14:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,489 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:14:29,490 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/C is initiating minor compaction (all files) 2024-12-08T00:14:29,490 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:14:29,490 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/C in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,490 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/138dac1b11cb46d888add886a922ef46, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/314df25b45234d4f946bd5d85d774b57, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a1aa6ccf67194add9092a437d48a4977, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/0404c929c6cc4b108acf9291306046de] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=48.7 K 2024-12-08T00:14:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:29,490 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 138dac1b11cb46d888add886a922ef46, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733616864010 2024-12-08T00:14:29,491 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 314df25b45234d4f946bd5d85d774b57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733616864663 2024-12-08T00:14:29,492 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a1aa6ccf67194add9092a437d48a4977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733616865345 2024-12-08T00:14:29,492 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 0404c929c6cc4b108acf9291306046de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733616865988 2024-12-08T00:14:29,494 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/A of 9938c46f9ca7a3abf34703707edea068 into 85b8ccd25a64458c812cea2f2a186f4c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:29,495 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:29,495 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/A, priority=12, startTime=1733616869447; duration=0sec 2024-12-08T00:14:29,495 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:29,495 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:A 2024-12-08T00:14:29,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/e683820b85d242d2a1168fea5d321731 is 50, key is test_row_0/A:col10/1733616868197/Put/seqid=0 2024-12-08T00:14:29,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741905_1081 (size=12301) 2024-12-08T00:14:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T00:14:29,502 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/e683820b85d242d2a1168fea5d321731 2024-12-08T00:14:29,511 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#C#compaction#69 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:29,512 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/38998bda0cf84da3892f58ec2d42a4d2 is 50, key is test_row_0/C:col10/1733616865988/Put/seqid=0 2024-12-08T00:14:29,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/f721c00f75db47c28c754d8115fab6ee is 50, key is test_row_0/B:col10/1733616868197/Put/seqid=0 2024-12-08T00:14:29,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741906_1082 (size=13051) 2024-12-08T00:14:29,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741907_1083 (size=12301) 2024-12-08T00:14:29,525 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/f721c00f75db47c28c754d8115fab6ee 2024-12-08T00:14:29,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/66c3e5a21ff14078b0227f2acb4c9331 is 50, key is test_row_0/C:col10/1733616868197/Put/seqid=0 2024-12-08T00:14:29,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741908_1084 (size=12301) 2024-12-08T00:14:29,549 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/66c3e5a21ff14078b0227f2acb4c9331 2024-12-08T00:14:29,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/e683820b85d242d2a1168fea5d321731 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e683820b85d242d2a1168fea5d321731 2024-12-08T00:14:29,559 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e683820b85d242d2a1168fea5d321731, entries=150, sequenceid=359, filesize=12.0 K 2024-12-08T00:14:29,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/f721c00f75db47c28c754d8115fab6ee as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/f721c00f75db47c28c754d8115fab6ee 2024-12-08T00:14:29,566 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/f721c00f75db47c28c754d8115fab6ee, entries=150, sequenceid=359, filesize=12.0 K 2024-12-08T00:14:29,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/66c3e5a21ff14078b0227f2acb4c9331 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/66c3e5a21ff14078b0227f2acb4c9331 2024-12-08T00:14:29,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/66c3e5a21ff14078b0227f2acb4c9331, entries=150, sequenceid=359, filesize=12.0 K 2024-12-08T00:14:29,576 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 9938c46f9ca7a3abf34703707edea068 in 87ms, sequenceid=359, compaction requested=false 2024-12-08T00:14:29,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:29,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:29,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-08T00:14:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-08T00:14:29,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-08T00:14:29,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 703 msec 2024-12-08T00:14:29,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 710 msec 2024-12-08T00:14:29,930 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/38998bda0cf84da3892f58ec2d42a4d2 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/38998bda0cf84da3892f58ec2d42a4d2 2024-12-08T00:14:29,936 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/C of 9938c46f9ca7a3abf34703707edea068 into 38998bda0cf84da3892f58ec2d42a4d2(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:29,936 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:29,936 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/C, priority=12, startTime=1733616869448; duration=0sec 2024-12-08T00:14:29,937 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:29,937 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:C 2024-12-08T00:14:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T00:14:30,011 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:30,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-08T00:14:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T00:14:30,013 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:30,014 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:30,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T00:14:30,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T00:14:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-08T00:14:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-08T00:14:30,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-08T00:14:30,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-08T00:14:30,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 159 msec 2024-12-08T00:14:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T00:14:30,331 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:30,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-08T00:14:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T00:14:30,334 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:30,335 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:30,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:30,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:14:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:30,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/cdaddb8a3bf44eb7bf6e7ceaccd6c705 is 50, key is test_row_0/A:col10/1733616870363/Put/seqid=0 2024-12-08T00:14:30,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741909_1085 (size=12301) 2024-12-08T00:14:30,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/cdaddb8a3bf44eb7bf6e7ceaccd6c705 2024-12-08T00:14:30,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/936e55893caf4d60a2f71bd7f4719339 is 50, key is test_row_0/B:col10/1733616870363/Put/seqid=0 2024-12-08T00:14:30,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741910_1086 (size=12301) 2024-12-08T00:14:30,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/936e55893caf4d60a2f71bd7f4719339 2024-12-08T00:14:30,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616870393, max timestamp=1733616870403Num uniques 859; 2024-12-08T00:14:30,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:30,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616870393, max timestamp=1733616870403Num uniques 859; 2024-12-08T00:14:30,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:30,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616870393, max timestamp=1733616870403Num uniques 859; 2024-12-08T00:14:30,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:30,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/3054c2729caf42a0ac7fb80271420588 is 50, key is test_row_0/C:col10/1733616870363/Put/seqid=0 2024-12-08T00:14:30,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741911_1087 (size=12301) 2024-12-08T00:14:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T00:14:30,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616930441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:30,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616930442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616930444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:30,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T00:14:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:30,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616930551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:30,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616930551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616930551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:30,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T00:14:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T00:14:30,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616930761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:30,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616930761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:30,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616930761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:30,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T00:14:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:30,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/3054c2729caf42a0ac7fb80271420588 2024-12-08T00:14:30,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/cdaddb8a3bf44eb7bf6e7ceaccd6c705 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/cdaddb8a3bf44eb7bf6e7ceaccd6c705 2024-12-08T00:14:30,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/cdaddb8a3bf44eb7bf6e7ceaccd6c705, entries=150, sequenceid=373, filesize=12.0 K 2024-12-08T00:14:30,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/936e55893caf4d60a2f71bd7f4719339 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/936e55893caf4d60a2f71bd7f4719339 2024-12-08T00:14:30,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/936e55893caf4d60a2f71bd7f4719339, entries=150, sequenceid=373, filesize=12.0 K 2024-12-08T00:14:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/3054c2729caf42a0ac7fb80271420588 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/3054c2729caf42a0ac7fb80271420588 2024-12-08T00:14:30,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/3054c2729caf42a0ac7fb80271420588, entries=150, sequenceid=373, filesize=12.0 K 2024-12-08T00:14:30,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9938c46f9ca7a3abf34703707edea068 in 502ms, sequenceid=373, compaction requested=true 2024-12-08T00:14:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:30,866 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:30,866 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:30,868 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:30,868 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:30,868 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/B is initiating minor compaction (all files) 2024-12-08T00:14:30,868 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/A is initiating minor compaction (all files) 2024-12-08T00:14:30,868 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/B in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,868 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/A in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,868 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/85b8ccd25a64458c812cea2f2a186f4c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e683820b85d242d2a1168fea5d321731, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/cdaddb8a3bf44eb7bf6e7ceaccd6c705] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=36.8 K 2024-12-08T00:14:30,868 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/945e81b728c84d83ab67cae7fea2278b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/f721c00f75db47c28c754d8115fab6ee, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/936e55893caf4d60a2f71bd7f4719339] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=36.8 K 2024-12-08T00:14:30,869 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 945e81b728c84d83ab67cae7fea2278b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733616865988 2024-12-08T00:14:30,869 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b8ccd25a64458c812cea2f2a186f4c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733616865988 2024-12-08T00:14:30,869 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting f721c00f75db47c28c754d8115fab6ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733616868193 2024-12-08T00:14:30,869 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting e683820b85d242d2a1168fea5d321731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733616868193 2024-12-08T00:14:30,869 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 936e55893caf4d60a2f71bd7f4719339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733616870358 2024-12-08T00:14:30,869 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdaddb8a3bf44eb7bf6e7ceaccd6c705, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733616870358 2024-12-08T00:14:30,878 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:30,878 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:30,878 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/92015271f62e4cc6a995abd2a387079c is 50, key is test_row_0/A:col10/1733616870363/Put/seqid=0 2024-12-08T00:14:30,878 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/90504a6be221408691bf2e3a7baaab99 is 50, key is test_row_0/B:col10/1733616870363/Put/seqid=0 2024-12-08T00:14:30,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741912_1088 (size=13153) 2024-12-08T00:14:30,892 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/90504a6be221408691bf2e3a7baaab99 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/90504a6be221408691bf2e3a7baaab99 2024-12-08T00:14:30,897 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/B of 9938c46f9ca7a3abf34703707edea068 into 90504a6be221408691bf2e3a7baaab99(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:30,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:30,897 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/B, priority=13, startTime=1733616870866; duration=0sec 2024-12-08T00:14:30,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:30,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:B 2024-12-08T00:14:30,898 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:30,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741913_1089 (size=13153) 2024-12-08T00:14:30,899 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:30,900 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/C is initiating minor compaction (all files) 2024-12-08T00:14:30,900 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/C in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,900 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/38998bda0cf84da3892f58ec2d42a4d2, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/66c3e5a21ff14078b0227f2acb4c9331, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/3054c2729caf42a0ac7fb80271420588] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=36.8 K 2024-12-08T00:14:30,900 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 38998bda0cf84da3892f58ec2d42a4d2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733616865988 2024-12-08T00:14:30,901 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 66c3e5a21ff14078b0227f2acb4c9331, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733616868193 2024-12-08T00:14:30,901 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 3054c2729caf42a0ac7fb80271420588, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733616870358 2024-12-08T00:14:30,907 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/92015271f62e4cc6a995abd2a387079c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/92015271f62e4cc6a995abd2a387079c 2024-12-08T00:14:30,912 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:30,913 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/08fea5ca85de40da83d1928b1624aca8 is 50, key is test_row_0/C:col10/1733616870363/Put/seqid=0 2024-12-08T00:14:30,914 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/A of 9938c46f9ca7a3abf34703707edea068 into 92015271f62e4cc6a995abd2a387079c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:30,914 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:30,914 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/A, priority=13, startTime=1733616870866; duration=0sec 2024-12-08T00:14:30,914 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:30,914 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:A 2024-12-08T00:14:30,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741914_1090 (size=13153) 2024-12-08T00:14:30,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T00:14:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:30,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-08T00:14:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/b98bc7e34abc4deeb53b22fd664da8bc is 50, key is test_row_0/A:col10/1733616870441/Put/seqid=0 2024-12-08T00:14:30,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741915_1091 (size=12301) 2024-12-08T00:14:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T00:14:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:31,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:31,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616931132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616931132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616931133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616931241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616931240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616931241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,326 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/08fea5ca85de40da83d1928b1624aca8 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/08fea5ca85de40da83d1928b1624aca8 2024-12-08T00:14:31,332 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/C of 9938c46f9ca7a3abf34703707edea068 into 08fea5ca85de40da83d1928b1624aca8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:31,332 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:31,332 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/C, priority=13, startTime=1733616870866; duration=0sec 2024-12-08T00:14:31,332 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:31,332 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:C 2024-12-08T00:14:31,357 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/b98bc7e34abc4deeb53b22fd664da8bc 2024-12-08T00:14:31,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/a18a2c85db3a4470ad521e62fa03d925 is 50, key is test_row_0/B:col10/1733616870441/Put/seqid=0 2024-12-08T00:14:31,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741916_1092 (size=12301) 2024-12-08T00:14:31,369 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/a18a2c85db3a4470ad521e62fa03d925 2024-12-08T00:14:31,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/fd89a951661a41019d20e1e1fc266563 is 50, key is test_row_0/C:col10/1733616870441/Put/seqid=0 2024-12-08T00:14:31,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741917_1093 (size=12301) 2024-12-08T00:14:31,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616931451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616931451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616931451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T00:14:31,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616931761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616931761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616931761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:31,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/fd89a951661a41019d20e1e1fc266563 2024-12-08T00:14:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/b98bc7e34abc4deeb53b22fd664da8bc as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b98bc7e34abc4deeb53b22fd664da8bc 2024-12-08T00:14:31,832 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b98bc7e34abc4deeb53b22fd664da8bc, entries=150, sequenceid=411, filesize=12.0 K 2024-12-08T00:14:31,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/a18a2c85db3a4470ad521e62fa03d925 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a18a2c85db3a4470ad521e62fa03d925 2024-12-08T00:14:31,839 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a18a2c85db3a4470ad521e62fa03d925, entries=150, sequenceid=411, filesize=12.0 K 2024-12-08T00:14:31,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/fd89a951661a41019d20e1e1fc266563 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/fd89a951661a41019d20e1e1fc266563 2024-12-08T00:14:31,846 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/fd89a951661a41019d20e1e1fc266563, entries=150, sequenceid=411, filesize=12.0 K 2024-12-08T00:14:31,847 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=67.09 KB/68700 for 9938c46f9ca7a3abf34703707edea068 in 900ms, sequenceid=411, compaction requested=false 2024-12-08T00:14:31,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:31,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:31,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-08T00:14:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-08T00:14:31,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-08T00:14:31,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5140 sec 2024-12-08T00:14:31,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.5200 sec 2024-12-08T00:14:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:32,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T00:14:32,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:32,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:32,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:32,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:32,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:32,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:32,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/f257654ece1341fb9ee95237cfc3a1a6 is 50, key is test_row_0/A:col10/1733616872271/Put/seqid=0 2024-12-08T00:14:32,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741918_1094 (size=14741) 2024-12-08T00:14:32,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616932290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616932296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616932297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616932400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616932400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616932401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T00:14:32,481 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:32,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-08T00:14:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T00:14:32,483 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:32,484 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:32,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T00:14:32,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616932618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616932618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616932620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T00:14:32,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:32,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:32,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:32,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:32,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:32,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/f257654ece1341fb9ee95237cfc3a1a6 2024-12-08T00:14:32,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/dc400fc479ad4c038e45a4695c9caece is 50, key is test_row_0/B:col10/1733616872271/Put/seqid=0 2024-12-08T00:14:32,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741919_1095 (size=12301) 2024-12-08T00:14:32,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/dc400fc479ad4c038e45a4695c9caece 2024-12-08T00:14:32,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/81716b7b18434c2ca7c12f7ebdfded01 is 50, key is test_row_0/C:col10/1733616872271/Put/seqid=0 2024-12-08T00:14:32,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741920_1096 (size=12301) 2024-12-08T00:14:32,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/81716b7b18434c2ca7c12f7ebdfded01 2024-12-08T00:14:32,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/f257654ece1341fb9ee95237cfc3a1a6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f257654ece1341fb9ee95237cfc3a1a6 2024-12-08T00:14:32,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f257654ece1341fb9ee95237cfc3a1a6, entries=200, sequenceid=426, filesize=14.4 K 2024-12-08T00:14:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/dc400fc479ad4c038e45a4695c9caece as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/dc400fc479ad4c038e45a4695c9caece 2024-12-08T00:14:32,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/dc400fc479ad4c038e45a4695c9caece, entries=150, sequenceid=426, filesize=12.0 K 2024-12-08T00:14:32,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/81716b7b18434c2ca7c12f7ebdfded01 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/81716b7b18434c2ca7c12f7ebdfded01 2024-12-08T00:14:32,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/81716b7b18434c2ca7c12f7ebdfded01, entries=150, sequenceid=426, filesize=12.0 K 2024-12-08T00:14:32,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9938c46f9ca7a3abf34703707edea068 in 482ms, sequenceid=426, compaction requested=true 2024-12-08T00:14:32,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:32,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:32,756 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:32,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:32,757 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:32,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:32,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:32,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:32,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:32,758 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:32,758 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:32,758 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/A is initiating minor compaction (all files) 2024-12-08T00:14:32,758 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/B is initiating minor compaction (all files) 2024-12-08T00:14:32,758 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/B in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:32,758 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/A in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:32,758 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/90504a6be221408691bf2e3a7baaab99, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a18a2c85db3a4470ad521e62fa03d925, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/dc400fc479ad4c038e45a4695c9caece] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=36.9 K 2024-12-08T00:14:32,758 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/92015271f62e4cc6a995abd2a387079c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b98bc7e34abc4deeb53b22fd664da8bc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f257654ece1341fb9ee95237cfc3a1a6] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=39.3 K 2024-12-08T00:14:32,759 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 90504a6be221408691bf2e3a7baaab99, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733616870358 2024-12-08T00:14:32,759 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92015271f62e4cc6a995abd2a387079c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733616870358 2024-12-08T00:14:32,759 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a18a2c85db3a4470ad521e62fa03d925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733616870430 2024-12-08T00:14:32,759 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting b98bc7e34abc4deeb53b22fd664da8bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733616870430 2024-12-08T00:14:32,759 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting dc400fc479ad4c038e45a4695c9caece, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733616871077 2024-12-08T00:14:32,759 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting f257654ece1341fb9ee95237cfc3a1a6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733616871077 2024-12-08T00:14:32,770 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:32,771 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/193ba75524fb4d56b899c7690893d16d is 50, key is test_row_0/A:col10/1733616872271/Put/seqid=0 2024-12-08T00:14:32,771 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:32,771 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/ee1324a0b3ab4720b8e52e4cffc55ed0 is 50, key is test_row_0/B:col10/1733616872271/Put/seqid=0 2024-12-08T00:14:32,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741921_1097 (size=13255) 2024-12-08T00:14:32,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741922_1098 (size=13255) 2024-12-08T00:14:32,783 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/ee1324a0b3ab4720b8e52e4cffc55ed0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ee1324a0b3ab4720b8e52e4cffc55ed0 2024-12-08T00:14:32,789 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/B of 9938c46f9ca7a3abf34703707edea068 into ee1324a0b3ab4720b8e52e4cffc55ed0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:32,789 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:32,789 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/B, priority=13, startTime=1733616872756; duration=0sec 2024-12-08T00:14:32,789 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:32,789 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:B 2024-12-08T00:14:32,789 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:32,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T00:14:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:32,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T00:14:32,791 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:32,791 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 9938c46f9ca7a3abf34703707edea068/C is initiating minor compaction (all files) 2024-12-08T00:14:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:32,791 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9938c46f9ca7a3abf34703707edea068/C in TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:32,791 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/08fea5ca85de40da83d1928b1624aca8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/fd89a951661a41019d20e1e1fc266563, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/81716b7b18434c2ca7c12f7ebdfded01] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp, totalSize=36.9 K 2024-12-08T00:14:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:32,792 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 08fea5ca85de40da83d1928b1624aca8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733616870358 2024-12-08T00:14:32,792 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting fd89a951661a41019d20e1e1fc266563, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733616870430 2024-12-08T00:14:32,793 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 81716b7b18434c2ca7c12f7ebdfded01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733616871077 2024-12-08T00:14:32,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/7974ba3e56fc40daa72aa0429e474754 is 50, key is test_row_0/A:col10/1733616872295/Put/seqid=0 2024-12-08T00:14:32,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741923_1099 (size=12301) 2024-12-08T00:14:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T00:14:32,803 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/7974ba3e56fc40daa72aa0429e474754 2024-12-08T00:14:32,814 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9938c46f9ca7a3abf34703707edea068#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:32,814 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/2a305bbe001545dc9501bfc73e3eb505 is 50, key is test_row_0/C:col10/1733616872271/Put/seqid=0 2024-12-08T00:14:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/39f9f9ddc7854d6aa7a648c288b1d576 is 50, key is test_row_0/B:col10/1733616872295/Put/seqid=0 2024-12-08T00:14:32,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741924_1100 (size=13255) 2024-12-08T00:14:32,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741925_1101 (size=12301) 2024-12-08T00:14:32,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. as already flushing 2024-12-08T00:14:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:32,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616932945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616932946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:32,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616932947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616933051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:33,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616933061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616933061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T00:14:33,182 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/193ba75524fb4d56b899c7690893d16d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/193ba75524fb4d56b899c7690893d16d 2024-12-08T00:14:33,188 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/A of 9938c46f9ca7a3abf34703707edea068 into 193ba75524fb4d56b899c7690893d16d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:33,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:33,188 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/A, priority=13, startTime=1733616872756; duration=0sec 2024-12-08T00:14:33,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:33,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:A 2024-12-08T00:14:33,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/39f9f9ddc7854d6aa7a648c288b1d576 2024-12-08T00:14:33,228 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/2a305bbe001545dc9501bfc73e3eb505 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/2a305bbe001545dc9501bfc73e3eb505 2024-12-08T00:14:33,234 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9938c46f9ca7a3abf34703707edea068/C of 9938c46f9ca7a3abf34703707edea068 into 2a305bbe001545dc9501bfc73e3eb505(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:33,234 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:33,234 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., storeName=9938c46f9ca7a3abf34703707edea068/C, priority=13, startTime=1733616872757; duration=0sec 2024-12-08T00:14:33,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/72eb28818a874c27804bc38f9d4dff38 is 50, key is test_row_0/C:col10/1733616872295/Put/seqid=0 2024-12-08T00:14:33,236 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:33,236 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:C 2024-12-08T00:14:33,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741926_1102 (size=12301) 2024-12-08T00:14:33,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:33,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616933261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616933271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616933280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1733616933570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38992 deadline: 1733616933581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39002 deadline: 1733616933591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:14:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T00:14:33,641 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/72eb28818a874c27804bc38f9d4dff38 2024-12-08T00:14:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/7974ba3e56fc40daa72aa0429e474754 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/7974ba3e56fc40daa72aa0429e474754 2024-12-08T00:14:33,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/7974ba3e56fc40daa72aa0429e474754, entries=150, sequenceid=445, filesize=12.0 K 2024-12-08T00:14:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/39f9f9ddc7854d6aa7a648c288b1d576 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/39f9f9ddc7854d6aa7a648c288b1d576 2024-12-08T00:14:33,656 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/39f9f9ddc7854d6aa7a648c288b1d576, entries=150, sequenceid=445, filesize=12.0 K 2024-12-08T00:14:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/72eb28818a874c27804bc38f9d4dff38 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/72eb28818a874c27804bc38f9d4dff38 2024-12-08T00:14:33,663 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/72eb28818a874c27804bc38f9d4dff38, entries=150, sequenceid=445, filesize=12.0 K 2024-12-08T00:14:33,664 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9938c46f9ca7a3abf34703707edea068 in 873ms, sequenceid=445, compaction requested=false 2024-12-08T00:14:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-08T00:14:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-08T00:14:33,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-08T00:14:33,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1810 sec 2024-12-08T00:14:33,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1870 sec 2024-12-08T00:14:33,923 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-08T00:14:33,923 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:14:33,923 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-08T00:14:33,923 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:14:33,924 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-08T00:14:33,924 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:14:33,927 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-08T00:14:33,927 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:14:33,927 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:33,927 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:33,927 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:33,927 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:33,928 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:33,928 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:33,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:33,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:33,928 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:33,928 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:33,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:33,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:34,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T00:14:34,083 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-08T00:14:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:34,084 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:14:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:34,084 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:34,084 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:34,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:34,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/b9003478a27b436386d6648b0f806445 is 50, key is test_row_0/A:col10/1733616874081/Put/seqid=0 2024-12-08T00:14:34,092 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-08T00:14:34,093 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:14:34,093 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,093 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:34,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741927_1103 (size=12301) 2024-12-08T00:14:34,102 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-08T00:14:34,102 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:14:34,102 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,102 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:34,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/b9003478a27b436386d6648b0f806445 2024-12-08T00:14:34,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/0db384f709204659b795e33846d2faaf is 50, key is test_row_0/B:col10/1733616874081/Put/seqid=0 2024-12-08T00:14:34,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741928_1104 (size=12301) 2024-12-08T00:14:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T00:14:34,632 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:34,774 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-08T00:14:34,774 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:14:34,774 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,775 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:34,835 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-08T00:14:34,835 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:14:34,836 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,836 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:34,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T00:14:34,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-12-08T00:14:34,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-08T00:14:34,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-08T00:14:34,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-08T00:14:34,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-08T00:14:34,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T00:14:34,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4225 2024-12-08T00:14:34,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4134 2024-12-08T00:14:34,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T00:14:34,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1879 2024-12-08T00:14:34,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5630 rows 2024-12-08T00:14:34,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1880 2024-12-08T00:14:34,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5627 rows 2024-12-08T00:14:34,840 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T00:14:34,840 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T00:14:34,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:34,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:14:34,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T00:14:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T00:14:34,855 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616874854"}]},"ts":"1733616874854"} 2024-12-08T00:14:34,857 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T00:14:34,857 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T00:14:34,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T00:14:34,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9938c46f9ca7a3abf34703707edea068, UNASSIGN}] 2024-12-08T00:14:34,863 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9938c46f9ca7a3abf34703707edea068, UNASSIGN 2024-12-08T00:14:34,865 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9938c46f9ca7a3abf34703707edea068, regionState=CLOSING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:34,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9938c46f9ca7a3abf34703707edea068, UNASSIGN because future has completed 2024-12-08T00:14:34,868 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T00:14:34,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:14:34,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/0db384f709204659b795e33846d2faaf 2024-12-08T00:14:34,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/06dda74de5e04eb091d443451ea29b28 is 50, key is test_row_0/C:col10/1733616874081/Put/seqid=0 2024-12-08T00:14:34,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741929_1105 (size=12301) 2024-12-08T00:14:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T00:14:35,027 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:35,028 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T00:14:35,029 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 9938c46f9ca7a3abf34703707edea068, disabling compactions & flushes 2024-12-08T00:14:35,030 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T00:14:35,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/06dda74de5e04eb091d443451ea29b28 2024-12-08T00:14:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/b9003478a27b436386d6648b0f806445 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b9003478a27b436386d6648b0f806445 2024-12-08T00:14:35,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b9003478a27b436386d6648b0f806445, entries=150, sequenceid=462, filesize=12.0 K 2024-12-08T00:14:35,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/0db384f709204659b795e33846d2faaf as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/0db384f709204659b795e33846d2faaf 2024-12-08T00:14:35,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/0db384f709204659b795e33846d2faaf, entries=150, sequenceid=462, filesize=12.0 K 2024-12-08T00:14:35,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/06dda74de5e04eb091d443451ea29b28 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/06dda74de5e04eb091d443451ea29b28 2024-12-08T00:14:35,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/06dda74de5e04eb091d443451ea29b28, entries=150, sequenceid=462, filesize=12.0 K 2024-12-08T00:14:35,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for 9938c46f9ca7a3abf34703707edea068 in 1285ms, sequenceid=462, compaction requested=true 2024-12-08T00:14:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9938c46f9ca7a3abf34703707edea068: 2024-12-08T00:14:35,368 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:35,368 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:35,368 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. after waiting 0 ms 2024-12-08T00:14:35,368 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:35,368 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. because compaction request was cancelled 2024-12-08T00:14:35,368 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. because compaction request was cancelled 2024-12-08T00:14:35,368 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:B 2024-12-08T00:14:35,368 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:A 2024-12-08T00:14:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:B, priority=-2147483648, current under compaction store size is 0 2024-12-08T00:14:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:35,368 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 9938c46f9ca7a3abf34703707edea068 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-08T00:14:35,368 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. because compaction request was cancelled 2024-12-08T00:14:35,368 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9938c46f9ca7a3abf34703707edea068:C 2024-12-08T00:14:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9938c46f9ca7a3abf34703707edea068:C, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:35,369 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=A 2024-12-08T00:14:35,369 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:35,369 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=B 2024-12-08T00:14:35,369 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:35,369 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9938c46f9ca7a3abf34703707edea068, store=C 2024-12-08T00:14:35,369 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:35,375 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/d23dfec7f3a2438986edf58647754e6c is 50, key is test_row_0/A:col10/1733616874832/Put/seqid=0 2024-12-08T00:14:35,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741930_1106 (size=9857) 2024-12-08T00:14:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T00:14:35,780 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/d23dfec7f3a2438986edf58647754e6c 2024-12-08T00:14:35,787 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/0b5bdcb30190444ba690b2839f5ba23c is 50, key is test_row_0/B:col10/1733616874832/Put/seqid=0 2024-12-08T00:14:35,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741931_1107 (size=9857) 2024-12-08T00:14:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T00:14:36,191 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/0b5bdcb30190444ba690b2839f5ba23c 2024-12-08T00:14:36,199 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/a23521c735e14a878b8e9e5e3265885b is 50, key is test_row_0/C:col10/1733616874832/Put/seqid=0 2024-12-08T00:14:36,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741932_1108 (size=9857) 2024-12-08T00:14:36,242 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T00:14:36,604 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/a23521c735e14a878b8e9e5e3265885b 2024-12-08T00:14:36,616 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/A/d23dfec7f3a2438986edf58647754e6c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d23dfec7f3a2438986edf58647754e6c 2024-12-08T00:14:36,622 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d23dfec7f3a2438986edf58647754e6c, entries=100, sequenceid=469, filesize=9.6 K 2024-12-08T00:14:36,623 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/B/0b5bdcb30190444ba690b2839f5ba23c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/0b5bdcb30190444ba690b2839f5ba23c 2024-12-08T00:14:36,628 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/0b5bdcb30190444ba690b2839f5ba23c, entries=100, sequenceid=469, filesize=9.6 K 2024-12-08T00:14:36,629 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/.tmp/C/a23521c735e14a878b8e9e5e3265885b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a23521c735e14a878b8e9e5e3265885b 2024-12-08T00:14:36,633 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a23521c735e14a878b8e9e5e3265885b, entries=100, sequenceid=469, filesize=9.6 K 2024-12-08T00:14:36,634 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9938c46f9ca7a3abf34703707edea068 in 1266ms, sequenceid=469, compaction requested=true 2024-12-08T00:14:36,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/93e92ba626c142b2b19df9069ffab0cc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f805deadf9ba4654b033e4b6caba8c10, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/5af0a7f6293e4cbf9c6572f5e1febad3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e6c496075ee74fcc9e340228b3b1da4f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/68c567277f2440eba58cbf472bc2ae50, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8968646c7a8841939998cbc5d3a8b590, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4419cf09051b40d4b3027813c63b19cf, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/90beaa7cede94a41885e1a76d55bac1e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/12d3cedc9ceb4fd8920a6d22daa2a6ce, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3fbd23a40e25420c9fb8f26727db43e6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4148452665324730b0c85d3b0c0a21a7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4831446ce420462b913f416032bf9a37, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b2770038c88c450b9dd4c7257efd3f67, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8a286f11bacb4a79b85947a0330cb12d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d66f6bcb7f3c48009e81cba53ed9212b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3426588e9fb24e2fab33df8451de69aa, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8e9b1c2bc0b248d1abbfd7852ddcd130, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/af22e6cb1c474d70a1022aaa6758b6b6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/c0f0bd6d9a854a7985cc25e5a9d6b9d5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e830a0d241a64777acf2bd4a78ecf9f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8c54ca65a0554832b5fac54a2a162e1e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/aa526fcf7c264a98876374d964539a7c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/85b8ccd25a64458c812cea2f2a186f4c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e683820b85d242d2a1168fea5d321731, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/92015271f62e4cc6a995abd2a387079c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/cdaddb8a3bf44eb7bf6e7ceaccd6c705, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b98bc7e34abc4deeb53b22fd664da8bc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f257654ece1341fb9ee95237cfc3a1a6] to archive 2024-12-08T00:14:36,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:14:36,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/93e92ba626c142b2b19df9069ffab0cc to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/93e92ba626c142b2b19df9069ffab0cc 2024-12-08T00:14:36,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f805deadf9ba4654b033e4b6caba8c10 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f805deadf9ba4654b033e4b6caba8c10 2024-12-08T00:14:36,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/5af0a7f6293e4cbf9c6572f5e1febad3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/5af0a7f6293e4cbf9c6572f5e1febad3 2024-12-08T00:14:36,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e6c496075ee74fcc9e340228b3b1da4f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e6c496075ee74fcc9e340228b3b1da4f 2024-12-08T00:14:36,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/68c567277f2440eba58cbf472bc2ae50 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/68c567277f2440eba58cbf472bc2ae50 2024-12-08T00:14:36,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8968646c7a8841939998cbc5d3a8b590 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8968646c7a8841939998cbc5d3a8b590 2024-12-08T00:14:36,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4419cf09051b40d4b3027813c63b19cf to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4419cf09051b40d4b3027813c63b19cf 2024-12-08T00:14:36,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/90beaa7cede94a41885e1a76d55bac1e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/90beaa7cede94a41885e1a76d55bac1e 2024-12-08T00:14:36,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/12d3cedc9ceb4fd8920a6d22daa2a6ce to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/12d3cedc9ceb4fd8920a6d22daa2a6ce 2024-12-08T00:14:36,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3fbd23a40e25420c9fb8f26727db43e6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3fbd23a40e25420c9fb8f26727db43e6 2024-12-08T00:14:36,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4148452665324730b0c85d3b0c0a21a7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4148452665324730b0c85d3b0c0a21a7 2024-12-08T00:14:36,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4831446ce420462b913f416032bf9a37 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/4831446ce420462b913f416032bf9a37 2024-12-08T00:14:36,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b2770038c88c450b9dd4c7257efd3f67 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b2770038c88c450b9dd4c7257efd3f67 2024-12-08T00:14:36,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8a286f11bacb4a79b85947a0330cb12d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8a286f11bacb4a79b85947a0330cb12d 2024-12-08T00:14:36,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d66f6bcb7f3c48009e81cba53ed9212b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d66f6bcb7f3c48009e81cba53ed9212b 2024-12-08T00:14:36,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3426588e9fb24e2fab33df8451de69aa to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/3426588e9fb24e2fab33df8451de69aa 2024-12-08T00:14:36,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8e9b1c2bc0b248d1abbfd7852ddcd130 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8e9b1c2bc0b248d1abbfd7852ddcd130 2024-12-08T00:14:36,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/af22e6cb1c474d70a1022aaa6758b6b6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/af22e6cb1c474d70a1022aaa6758b6b6 2024-12-08T00:14:36,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/c0f0bd6d9a854a7985cc25e5a9d6b9d5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/c0f0bd6d9a854a7985cc25e5a9d6b9d5 2024-12-08T00:14:36,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e830a0d241a64777acf2bd4a78ecf9f7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e830a0d241a64777acf2bd4a78ecf9f7 2024-12-08T00:14:36,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8c54ca65a0554832b5fac54a2a162e1e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/8c54ca65a0554832b5fac54a2a162e1e 2024-12-08T00:14:36,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/aa526fcf7c264a98876374d964539a7c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/aa526fcf7c264a98876374d964539a7c 2024-12-08T00:14:36,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/85b8ccd25a64458c812cea2f2a186f4c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/85b8ccd25a64458c812cea2f2a186f4c 2024-12-08T00:14:36,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e683820b85d242d2a1168fea5d321731 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/e683820b85d242d2a1168fea5d321731 2024-12-08T00:14:36,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/92015271f62e4cc6a995abd2a387079c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/92015271f62e4cc6a995abd2a387079c 2024-12-08T00:14:36,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/cdaddb8a3bf44eb7bf6e7ceaccd6c705 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/cdaddb8a3bf44eb7bf6e7ceaccd6c705 2024-12-08T00:14:36,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b98bc7e34abc4deeb53b22fd664da8bc to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b98bc7e34abc4deeb53b22fd664da8bc 2024-12-08T00:14:36,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f257654ece1341fb9ee95237cfc3a1a6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/f257654ece1341fb9ee95237cfc3a1a6 2024-12-08T00:14:36,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/98e6f38c9b0e46098eddeb92e0c39e3d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a2eca89c463d4ec29696725b588993db, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e944790b1e1843ed81465021c1d6ef29, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/253df10505ed4868942279b4ab2335f0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/2b2bbf1ca3714dc9807104173671f625, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1191795e83b64d8ea14cf1fb8692291d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d26beab8adb434eb44f1aef27ddccc9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ba529cc32e7d408b9bb69cc5698d17ad, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1e4609b0523d442d808d8e248c1d2201, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/9513116425484938a8cb87e5d4145934, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d12b255daca457daf2077e54c53aa08, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e464b227259745d4b1548e8b9af7f35e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/5dfbe24f4a5b45f7b5447ef9ed6ce132, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ee132481fbd0443d9990ef8f98da9c8a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/205e031b99f247728294cdb79bdc5d79, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/08132d8bd2cd4adba0dcc8527adc275a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/71ced269e0ca4b80ad979ab7df38e41f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/6705e62516e24615bd8e30f7b9f708d6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/d0777084916642f3b7edb4c98fad72d0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/388e679f65524a81a291f6e360d71625, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/38281a9b47bb4adb908bb6d2b56c71e7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/945e81b728c84d83ab67cae7fea2278b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/3cffe53dd3b24c55b9a0a0034a961919, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/f721c00f75db47c28c754d8115fab6ee, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/90504a6be221408691bf2e3a7baaab99, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/936e55893caf4d60a2f71bd7f4719339, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a18a2c85db3a4470ad521e62fa03d925, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/dc400fc479ad4c038e45a4695c9caece] to archive 2024-12-08T00:14:36,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:14:36,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/98e6f38c9b0e46098eddeb92e0c39e3d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/98e6f38c9b0e46098eddeb92e0c39e3d 2024-12-08T00:14:36,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a2eca89c463d4ec29696725b588993db to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a2eca89c463d4ec29696725b588993db 2024-12-08T00:14:36,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e944790b1e1843ed81465021c1d6ef29 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e944790b1e1843ed81465021c1d6ef29 2024-12-08T00:14:36,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/253df10505ed4868942279b4ab2335f0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/253df10505ed4868942279b4ab2335f0 2024-12-08T00:14:36,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/2b2bbf1ca3714dc9807104173671f625 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/2b2bbf1ca3714dc9807104173671f625 2024-12-08T00:14:36,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1191795e83b64d8ea14cf1fb8692291d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1191795e83b64d8ea14cf1fb8692291d 2024-12-08T00:14:36,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d26beab8adb434eb44f1aef27ddccc9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d26beab8adb434eb44f1aef27ddccc9 2024-12-08T00:14:36,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ba529cc32e7d408b9bb69cc5698d17ad to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ba529cc32e7d408b9bb69cc5698d17ad 2024-12-08T00:14:36,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1e4609b0523d442d808d8e248c1d2201 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/1e4609b0523d442d808d8e248c1d2201 2024-12-08T00:14:36,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/9513116425484938a8cb87e5d4145934 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/9513116425484938a8cb87e5d4145934 2024-12-08T00:14:36,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d12b255daca457daf2077e54c53aa08 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/8d12b255daca457daf2077e54c53aa08 2024-12-08T00:14:36,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e464b227259745d4b1548e8b9af7f35e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/e464b227259745d4b1548e8b9af7f35e 2024-12-08T00:14:36,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/5dfbe24f4a5b45f7b5447ef9ed6ce132 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/5dfbe24f4a5b45f7b5447ef9ed6ce132 2024-12-08T00:14:36,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ee132481fbd0443d9990ef8f98da9c8a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ee132481fbd0443d9990ef8f98da9c8a 2024-12-08T00:14:36,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/205e031b99f247728294cdb79bdc5d79 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/205e031b99f247728294cdb79bdc5d79 2024-12-08T00:14:36,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/08132d8bd2cd4adba0dcc8527adc275a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/08132d8bd2cd4adba0dcc8527adc275a 2024-12-08T00:14:36,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/71ced269e0ca4b80ad979ab7df38e41f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/71ced269e0ca4b80ad979ab7df38e41f 2024-12-08T00:14:36,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/6705e62516e24615bd8e30f7b9f708d6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/6705e62516e24615bd8e30f7b9f708d6 2024-12-08T00:14:36,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/d0777084916642f3b7edb4c98fad72d0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/d0777084916642f3b7edb4c98fad72d0 2024-12-08T00:14:36,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/388e679f65524a81a291f6e360d71625 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/388e679f65524a81a291f6e360d71625 2024-12-08T00:14:36,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/38281a9b47bb4adb908bb6d2b56c71e7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/38281a9b47bb4adb908bb6d2b56c71e7 2024-12-08T00:14:36,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/945e81b728c84d83ab67cae7fea2278b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/945e81b728c84d83ab67cae7fea2278b 2024-12-08T00:14:36,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/3cffe53dd3b24c55b9a0a0034a961919 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/3cffe53dd3b24c55b9a0a0034a961919 2024-12-08T00:14:36,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/f721c00f75db47c28c754d8115fab6ee to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/f721c00f75db47c28c754d8115fab6ee 2024-12-08T00:14:36,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/90504a6be221408691bf2e3a7baaab99 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/90504a6be221408691bf2e3a7baaab99 2024-12-08T00:14:36,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/936e55893caf4d60a2f71bd7f4719339 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/936e55893caf4d60a2f71bd7f4719339 2024-12-08T00:14:36,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a18a2c85db3a4470ad521e62fa03d925 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/a18a2c85db3a4470ad521e62fa03d925 2024-12-08T00:14:36,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/dc400fc479ad4c038e45a4695c9caece to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/dc400fc479ad4c038e45a4695c9caece 2024-12-08T00:14:36,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7b6b850f7515469d82320d0938787310, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/5cb139be12ca4b938e71e3ed241d91af, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7392da633af3454f9a7ee5e07b802f24, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/437b338517b445db8d11575d10713141, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/125d6532c359489c86dbef983f7c9191, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c630d29f70f24b9ca2cdf8fd1cf1143c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f05e3dfb3b754549b87066c6b9c0547d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/96f829f08229456391a1970bb37a2091, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/e4db68d6bc6c4f0d9f7a5123dbe32c64, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7cd1471a8c434ce9b6c4c3a383fad3a5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c1104ffef3a94f4a99c3c8efda4ca87a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7d343654c25c4c4cacfd828491e8e579, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c450910dd1a04729ba306c4ca072fdd1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/6267782f7d9649ee8c435f1b12959fdc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f137d857fd7a4695b4529dfdaa04b813, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/4d3667ddf0624aabb5369f9f97bc5c05, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/d2a965ac80b546ccbd3d013731718c11, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/138dac1b11cb46d888add886a922ef46, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/ea7158b411e143a4adb2dd619a79cc17, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/314df25b45234d4f946bd5d85d774b57, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a1aa6ccf67194add9092a437d48a4977, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/38998bda0cf84da3892f58ec2d42a4d2, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/0404c929c6cc4b108acf9291306046de, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/66c3e5a21ff14078b0227f2acb4c9331, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/08fea5ca85de40da83d1928b1624aca8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/3054c2729caf42a0ac7fb80271420588, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/fd89a951661a41019d20e1e1fc266563, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/81716b7b18434c2ca7c12f7ebdfded01] to archive 2024-12-08T00:14:36,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:14:36,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7b6b850f7515469d82320d0938787310 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7b6b850f7515469d82320d0938787310 2024-12-08T00:14:36,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/5cb139be12ca4b938e71e3ed241d91af to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/5cb139be12ca4b938e71e3ed241d91af 2024-12-08T00:14:36,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7392da633af3454f9a7ee5e07b802f24 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7392da633af3454f9a7ee5e07b802f24 2024-12-08T00:14:36,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/437b338517b445db8d11575d10713141 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/437b338517b445db8d11575d10713141 2024-12-08T00:14:36,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/125d6532c359489c86dbef983f7c9191 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/125d6532c359489c86dbef983f7c9191 2024-12-08T00:14:36,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c630d29f70f24b9ca2cdf8fd1cf1143c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c630d29f70f24b9ca2cdf8fd1cf1143c 2024-12-08T00:14:36,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f05e3dfb3b754549b87066c6b9c0547d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f05e3dfb3b754549b87066c6b9c0547d 2024-12-08T00:14:36,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/96f829f08229456391a1970bb37a2091 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/96f829f08229456391a1970bb37a2091 2024-12-08T00:14:36,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/e4db68d6bc6c4f0d9f7a5123dbe32c64 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/e4db68d6bc6c4f0d9f7a5123dbe32c64 2024-12-08T00:14:36,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7cd1471a8c434ce9b6c4c3a383fad3a5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7cd1471a8c434ce9b6c4c3a383fad3a5 2024-12-08T00:14:36,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c1104ffef3a94f4a99c3c8efda4ca87a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c1104ffef3a94f4a99c3c8efda4ca87a 2024-12-08T00:14:36,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7d343654c25c4c4cacfd828491e8e579 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/7d343654c25c4c4cacfd828491e8e579 2024-12-08T00:14:36,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c450910dd1a04729ba306c4ca072fdd1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/c450910dd1a04729ba306c4ca072fdd1 2024-12-08T00:14:36,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/6267782f7d9649ee8c435f1b12959fdc to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/6267782f7d9649ee8c435f1b12959fdc 2024-12-08T00:14:36,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f137d857fd7a4695b4529dfdaa04b813 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/f137d857fd7a4695b4529dfdaa04b813 2024-12-08T00:14:36,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/4d3667ddf0624aabb5369f9f97bc5c05 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/4d3667ddf0624aabb5369f9f97bc5c05 2024-12-08T00:14:36,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/d2a965ac80b546ccbd3d013731718c11 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/d2a965ac80b546ccbd3d013731718c11 2024-12-08T00:14:36,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/138dac1b11cb46d888add886a922ef46 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/138dac1b11cb46d888add886a922ef46 2024-12-08T00:14:36,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/ea7158b411e143a4adb2dd619a79cc17 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/ea7158b411e143a4adb2dd619a79cc17 2024-12-08T00:14:36,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/314df25b45234d4f946bd5d85d774b57 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/314df25b45234d4f946bd5d85d774b57 2024-12-08T00:14:36,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a1aa6ccf67194add9092a437d48a4977 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a1aa6ccf67194add9092a437d48a4977 2024-12-08T00:14:36,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/38998bda0cf84da3892f58ec2d42a4d2 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/38998bda0cf84da3892f58ec2d42a4d2 2024-12-08T00:14:36,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/0404c929c6cc4b108acf9291306046de to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/0404c929c6cc4b108acf9291306046de 2024-12-08T00:14:36,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/66c3e5a21ff14078b0227f2acb4c9331 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/66c3e5a21ff14078b0227f2acb4c9331 2024-12-08T00:14:36,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/08fea5ca85de40da83d1928b1624aca8 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/08fea5ca85de40da83d1928b1624aca8 2024-12-08T00:14:36,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/3054c2729caf42a0ac7fb80271420588 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/3054c2729caf42a0ac7fb80271420588 2024-12-08T00:14:36,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/fd89a951661a41019d20e1e1fc266563 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/fd89a951661a41019d20e1e1fc266563 2024-12-08T00:14:36,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/81716b7b18434c2ca7c12f7ebdfded01 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/81716b7b18434c2ca7c12f7ebdfded01 2024-12-08T00:14:36,752 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/recovered.edits/472.seqid, newMaxSeqId=472, maxSeqId=1 2024-12-08T00:14:36,753 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. 2024-12-08T00:14:36,753 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 9938c46f9ca7a3abf34703707edea068: Waiting for close lock at 1733616875029Running coprocessor pre-close hooks at 1733616875029Disabling compacts and flushes for region at 1733616875029Disabling writes for close at 1733616875368 (+339 ms)Obtaining lock to block concurrent updates at 1733616875368Preparing flush snapshotting stores in 9938c46f9ca7a3abf34703707edea068 at 1733616875368Finished memstore snapshotting TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733616875369 (+1 ms)Flushing stores of TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068. at 1733616875370 (+1 ms)Flushing 9938c46f9ca7a3abf34703707edea068/A: creating writer at 1733616875370Flushing 9938c46f9ca7a3abf34703707edea068/A: appending metadata at 1733616875374 (+4 ms)Flushing 9938c46f9ca7a3abf34703707edea068/A: closing flushed file at 1733616875374Flushing 9938c46f9ca7a3abf34703707edea068/B: creating writer at 1733616875784 (+410 ms)Flushing 9938c46f9ca7a3abf34703707edea068/B: appending metadata at 1733616875787 (+3 ms)Flushing 9938c46f9ca7a3abf34703707edea068/B: closing flushed file at 1733616875787Flushing 9938c46f9ca7a3abf34703707edea068/C: creating writer at 1733616876196 (+409 ms)Flushing 9938c46f9ca7a3abf34703707edea068/C: appending metadata at 1733616876198 (+2 ms)Flushing 9938c46f9ca7a3abf34703707edea068/C: closing flushed file at 1733616876198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ae42591: reopening flushed file at 1733616876614 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8c845a9: reopening flushed file at 1733616876622 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@506d003f: reopening flushed file at 1733616876628 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9938c46f9ca7a3abf34703707edea068 in 1266ms, sequenceid=469, compaction requested=true at 1733616876634 (+6 ms)Writing region close event to WAL at 1733616876748 (+114 ms)Running coprocessor post-close hooks at 1733616876753 (+5 ms)Closed at 1733616876753 2024-12-08T00:14:36,756 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:36,756 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9938c46f9ca7a3abf34703707edea068, regionState=CLOSED 2024-12-08T00:14:36,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:14:36,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-08T00:14:36,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 9938c46f9ca7a3abf34703707edea068, server=c1a8cf8d4cfa,36021,1733616847970 in 1.8910 sec 2024-12-08T00:14:36,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-08T00:14:36,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9938c46f9ca7a3abf34703707edea068, UNASSIGN in 1.8980 sec 2024-12-08T00:14:36,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-08T00:14:36,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9050 sec 2024-12-08T00:14:36,765 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616876765"}]},"ts":"1733616876765"} 2024-12-08T00:14:36,766 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T00:14:36,766 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T00:14:36,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9180 sec 2024-12-08T00:14:37,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T00:14:37,012 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:37,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T00:14:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:37,024 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T00:14:37,026 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:37,032 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:37,038 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/recovered.edits] 2024-12-08T00:14:37,042 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/193ba75524fb4d56b899c7690893d16d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/193ba75524fb4d56b899c7690893d16d 2024-12-08T00:14:37,043 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/7974ba3e56fc40daa72aa0429e474754 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/7974ba3e56fc40daa72aa0429e474754 2024-12-08T00:14:37,045 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b9003478a27b436386d6648b0f806445 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/b9003478a27b436386d6648b0f806445 2024-12-08T00:14:37,046 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d23dfec7f3a2438986edf58647754e6c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/A/d23dfec7f3a2438986edf58647754e6c 2024-12-08T00:14:37,049 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/0b5bdcb30190444ba690b2839f5ba23c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/0b5bdcb30190444ba690b2839f5ba23c 2024-12-08T00:14:37,050 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/0db384f709204659b795e33846d2faaf to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/0db384f709204659b795e33846d2faaf 2024-12-08T00:14:37,051 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/39f9f9ddc7854d6aa7a648c288b1d576 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/39f9f9ddc7854d6aa7a648c288b1d576 2024-12-08T00:14:37,053 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ee1324a0b3ab4720b8e52e4cffc55ed0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/B/ee1324a0b3ab4720b8e52e4cffc55ed0 2024-12-08T00:14:37,055 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/06dda74de5e04eb091d443451ea29b28 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/06dda74de5e04eb091d443451ea29b28 2024-12-08T00:14:37,056 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/2a305bbe001545dc9501bfc73e3eb505 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/2a305bbe001545dc9501bfc73e3eb505 2024-12-08T00:14:37,057 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/72eb28818a874c27804bc38f9d4dff38 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/72eb28818a874c27804bc38f9d4dff38 2024-12-08T00:14:37,058 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a23521c735e14a878b8e9e5e3265885b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/C/a23521c735e14a878b8e9e5e3265885b 2024-12-08T00:14:37,061 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/recovered.edits/472.seqid to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068/recovered.edits/472.seqid 2024-12-08T00:14:37,062 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/9938c46f9ca7a3abf34703707edea068 2024-12-08T00:14:37,062 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T00:14:37,066 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:37,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-08T00:14:37,074 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T00:14:37,100 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T00:14:37,102 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:37,102 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T00:14:37,102 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733616877102"}]},"ts":"9223372036854775807"} 2024-12-08T00:14:37,107 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T00:14:37,107 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9938c46f9ca7a3abf34703707edea068, NAME => 'TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T00:14:37,107 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T00:14:37,107 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733616877107"}]},"ts":"9223372036854775807"} 2024-12-08T00:14:37,110 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T00:14:37,111 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:37,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-12-08T00:14:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T00:14:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T00:14:37,132 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:37,142 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1076213060_22 at /127.0.0.1:53814 [Waiting for operation #307] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1975563135_22 at /127.0.0.1:44626 [Waiting for operation #294] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=400 (was 321) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8352 (was 8983) 2024-12-08T00:14:37,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=400, ProcessCount=11, AvailableMemoryMB=8350 2024-12-08T00:14:37,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T00:14:37,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:14:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:37,156 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T00:14:37,156 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:37,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-08T00:14:37,157 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T00:14:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T00:14:37,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741933_1109 (size=1061) 2024-12-08T00:14:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T00:14:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T00:14:37,568 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329 2024-12-08T00:14:37,576 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T00:14:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741934_1110 (size=53) 2024-12-08T00:14:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T00:14:37,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:14:37,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c31d593296c0be255698232087c08858, disabling compactions & flushes 2024-12-08T00:14:37,983 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:37,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:37,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. after waiting 0 ms 2024-12-08T00:14:37,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:37,984 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:37,984 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c31d593296c0be255698232087c08858: Waiting for close lock at 1733616877983Disabling compacts and flushes for region at 1733616877983Disabling writes for close at 1733616877983Writing region close event to WAL at 1733616877983Closed at 1733616877983 2024-12-08T00:14:37,986 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T00:14:37,987 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733616877986"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733616877986"}]},"ts":"1733616877986"} 2024-12-08T00:14:37,989 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T00:14:37,990 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T00:14:37,990 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616877990"}]},"ts":"1733616877990"} 2024-12-08T00:14:37,992 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T00:14:37,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, ASSIGN}] 2024-12-08T00:14:37,994 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, ASSIGN 2024-12-08T00:14:37,995 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, ASSIGN; state=OFFLINE, location=c1a8cf8d4cfa,36021,1733616847970; forceNewPlan=false, retain=false 2024-12-08T00:14:38,146 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=c31d593296c0be255698232087c08858, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:38,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, ASSIGN because future has completed 2024-12-08T00:14:38,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:14:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T00:14:38,310 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:38,310 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:14:38,310 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c31d593296c0be255698232087c08858 2024-12-08T00:14:38,310 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:14:38,310 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for c31d593296c0be255698232087c08858 2024-12-08T00:14:38,310 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for c31d593296c0be255698232087c08858 2024-12-08T00:14:38,312 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c31d593296c0be255698232087c08858 2024-12-08T00:14:38,313 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:14:38,314 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c31d593296c0be255698232087c08858 columnFamilyName A 2024-12-08T00:14:38,314 DEBUG [StoreOpener-c31d593296c0be255698232087c08858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:38,314 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(327): Store=c31d593296c0be255698232087c08858/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:38,315 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c31d593296c0be255698232087c08858 2024-12-08T00:14:38,316 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:14:38,316 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c31d593296c0be255698232087c08858 columnFamilyName B 2024-12-08T00:14:38,316 DEBUG [StoreOpener-c31d593296c0be255698232087c08858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:38,316 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(327): Store=c31d593296c0be255698232087c08858/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:38,316 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c31d593296c0be255698232087c08858 2024-12-08T00:14:38,317 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:14:38,318 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c31d593296c0be255698232087c08858 columnFamilyName C 2024-12-08T00:14:38,318 DEBUG [StoreOpener-c31d593296c0be255698232087c08858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:38,318 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(327): Store=c31d593296c0be255698232087c08858/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:38,318 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:38,318 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for c31d593296c0be255698232087c08858 2024-12-08T00:14:38,319 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858 2024-12-08T00:14:38,319 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858 2024-12-08T00:14:38,320 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for c31d593296c0be255698232087c08858 2024-12-08T00:14:38,320 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for c31d593296c0be255698232087c08858 2024-12-08T00:14:38,320 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:14:38,322 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for c31d593296c0be255698232087c08858 2024-12-08T00:14:38,323 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T00:14:38,324 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened c31d593296c0be255698232087c08858; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69466643, jitterRate=0.03513364493846893}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:14:38,324 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c31d593296c0be255698232087c08858 2024-12-08T00:14:38,325 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for c31d593296c0be255698232087c08858: Running coprocessor pre-open hook at 1733616878310Writing region info on filesystem at 1733616878310Initializing all the Stores at 1733616878311 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616878311Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616878312 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616878312Cleaning up temporary data from old regions at 1733616878320 (+8 ms)Running coprocessor post-open hooks at 1733616878324 (+4 ms)Region opened successfully at 1733616878325 (+1 ms) 2024-12-08T00:14:38,326 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., pid=34, masterSystemTime=1733616878307 2024-12-08T00:14:38,328 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:38,328 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:38,328 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=c31d593296c0be255698232087c08858, regionState=OPEN, openSeqNum=2, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:38,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:14:38,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-08T00:14:38,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 in 179 msec 2024-12-08T00:14:38,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-08T00:14:38,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, ASSIGN in 340 msec 2024-12-08T00:14:38,336 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T00:14:38,336 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616878336"}]},"ts":"1733616878336"} 2024-12-08T00:14:38,338 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T00:14:38,339 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T00:14:38,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-12-08T00:14:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T00:14:39,302 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:39,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcb2494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:39,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:39,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:39,305 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af5b370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:39,308 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:39,309 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:39,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:39,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T00:14:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:14:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-12-08T00:14:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T00:14:39,317 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T00:14:39,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T00:14:39,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:14:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T00:14:39,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741935_1111 (size=1097) 2024-12-08T00:14:39,747 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T00:14:39,748 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T00:14:39,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T00:14:39,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, REOPEN/MOVE}] 2024-12-08T00:14:39,762 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, REOPEN/MOVE 2024-12-08T00:14:39,763 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=c31d593296c0be255698232087c08858, regionState=CLOSING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:39,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, REOPEN/MOVE because future has completed 2024-12-08T00:14:39,765 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T00:14:39,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:14:39,918 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close c31d593296c0be255698232087c08858 2024-12-08T00:14:39,918 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T00:14:39,918 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing c31d593296c0be255698232087c08858, disabling compactions & flushes 2024-12-08T00:14:39,918 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:39,918 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:39,918 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. after waiting 0 ms 2024-12-08T00:14:39,918 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:39,922 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T00:14:39,922 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:39,922 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for c31d593296c0be255698232087c08858: Waiting for close lock at 1733616879918Running coprocessor pre-close hooks at 1733616879918Disabling compacts and flushes for region at 1733616879918Disabling writes for close at 1733616879918Writing region close event to WAL at 1733616879918Running coprocessor post-close hooks at 1733616879922 (+4 ms)Closed at 1733616879922 2024-12-08T00:14:39,923 WARN [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: c31d593296c0be255698232087c08858 to self. 2024-12-08T00:14:39,925 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed c31d593296c0be255698232087c08858 2024-12-08T00:14:39,926 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=c31d593296c0be255698232087c08858, regionState=CLOSED 2024-12-08T00:14:39,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:14:39,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-08T00:14:39,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 in 164 msec 2024-12-08T00:14:39,933 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, REOPEN/MOVE; state=CLOSED, location=c1a8cf8d4cfa,36021,1733616847970; forceNewPlan=false, retain=true 2024-12-08T00:14:40,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=c31d593296c0be255698232087c08858, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, REOPEN/MOVE because future has completed 2024-12-08T00:14:40,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:14:40,247 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:40,247 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:14:40,247 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c31d593296c0be255698232087c08858 2024-12-08T00:14:40,247 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:14:40,248 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for c31d593296c0be255698232087c08858 2024-12-08T00:14:40,248 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for c31d593296c0be255698232087c08858 2024-12-08T00:14:40,251 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c31d593296c0be255698232087c08858 2024-12-08T00:14:40,252 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:14:40,257 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c31d593296c0be255698232087c08858 columnFamilyName A 2024-12-08T00:14:40,259 DEBUG [StoreOpener-c31d593296c0be255698232087c08858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:40,260 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(327): Store=c31d593296c0be255698232087c08858/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:40,260 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c31d593296c0be255698232087c08858 2024-12-08T00:14:40,261 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:14:40,261 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c31d593296c0be255698232087c08858 columnFamilyName B 2024-12-08T00:14:40,261 DEBUG [StoreOpener-c31d593296c0be255698232087c08858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:40,262 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(327): Store=c31d593296c0be255698232087c08858/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:40,262 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c31d593296c0be255698232087c08858 2024-12-08T00:14:40,263 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:14:40,263 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c31d593296c0be255698232087c08858 columnFamilyName C 2024-12-08T00:14:40,263 DEBUG [StoreOpener-c31d593296c0be255698232087c08858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:40,263 INFO [StoreOpener-c31d593296c0be255698232087c08858-1 {}] regionserver.HStore(327): Store=c31d593296c0be255698232087c08858/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:14:40,263 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:40,264 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for c31d593296c0be255698232087c08858 2024-12-08T00:14:40,264 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858 2024-12-08T00:14:40,265 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858 2024-12-08T00:14:40,266 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for c31d593296c0be255698232087c08858 2024-12-08T00:14:40,266 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for c31d593296c0be255698232087c08858 2024-12-08T00:14:40,267 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:14:40,268 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for c31d593296c0be255698232087c08858 2024-12-08T00:14:40,268 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened c31d593296c0be255698232087c08858; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67877042, jitterRate=0.011446744203567505}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:14:40,268 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c31d593296c0be255698232087c08858 2024-12-08T00:14:40,269 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for c31d593296c0be255698232087c08858: Running coprocessor pre-open hook at 1733616880248Writing region info on filesystem at 1733616880248Initializing all the Stores at 1733616880248Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616880249 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616880249Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616880249Cleaning up temporary data from old regions at 1733616880266 (+17 ms)Running coprocessor post-open hooks at 1733616880268 (+2 ms)Region opened successfully at 1733616880269 (+1 ms) 2024-12-08T00:14:40,270 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., pid=39, masterSystemTime=1733616880244 2024-12-08T00:14:40,272 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:40,272 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:40,272 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=c31d593296c0be255698232087c08858, regionState=OPEN, openSeqNum=5, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:14:40,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-08T00:14:40,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 in 184 msec 2024-12-08T00:14:40,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-08T00:14:40,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, REOPEN/MOVE in 514 msec 2024-12-08T00:14:40,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-08T00:14:40,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-12-08T00:14:40,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-12-08T00:14:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-08T00:14:40,285 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:40,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:40,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:40,287 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59587148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:40,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d1d783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:40,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:40,290 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:40,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:40,291 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:40,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5237b5e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:40,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:40,293 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@124a9e36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:40,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58fbbcb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:40,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:40,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d8f9fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:40,298 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4445ae64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:40,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:40,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4f6f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:40,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24080fea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:40,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:40,304 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4343bbac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:40,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e65ac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:40,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:40,308 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761a8232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:40,310 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:40,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:40,313 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed9cd7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:40,314 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:40,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:14:40,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:14:40,316 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:14:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:14:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:14:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75df7a48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:14:40,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:14:40,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:14:40,317 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:14:40,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfa90e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:14:40,319 DEBUG [Thread-574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:40,320 DEBUG [Thread-580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:40,320 DEBUG [Thread-578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:40,320 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:40,321 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:40,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:40,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:40,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:40,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:40,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:40,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:40,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:40,324 DEBUG [Thread-585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-08T00:14:40,325 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T00:14:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:14:40,326 DEBUG [Thread-576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:40,326 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:40,326 DEBUG [Thread-582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:14:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:40,327 DEBUG [Thread-582 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:40,328 DEBUG [Thread-582 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:14:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:14:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:14:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:40,329 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:40,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:14:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:14:40,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:14:40,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:40,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:40,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:14:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:14:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:14:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:14:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:14:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:40,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:14:40,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:40,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:40,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:40,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:40,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:40,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:40,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30174, totalHeapSize=68776, min timestamp=1733616880345, max timestamp=1733616880382Num uniques 859; 2024-12-08T00:14:40,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=30174, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-12-08T00:14:40,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30174, totalHeapSize=68776, min timestamp=1733616880345, max timestamp=1733616880382Num uniques 859; 2024-12-08T00:14:40,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=30174, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-12-08T00:14:40,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120803b43610eb42460897ad8f3ea6663f4f_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616880339/Put/seqid=0 2024-12-08T00:14:40,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30174, totalHeapSize=68776, min timestamp=1733616880345, max timestamp=1733616880382Num uniques 859; 2024-12-08T00:14:40,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=30174, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-12-08T00:14:40,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616940403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616940403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616940405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616940407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741936_1112 (size=19474) 2024-12-08T00:14:40,419 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:40,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616940411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,427 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120803b43610eb42460897ad8f3ea6663f4f_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120803b43610eb42460897ad8f3ea6663f4f_c31d593296c0be255698232087c08858 2024-12-08T00:14:40,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/92c54d1427ef4eeea8d587f3732d30f7, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T00:14:40,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/92c54d1427ef4eeea8d587f3732d30f7 is 175, key is test_row_0/A:col10/1733616880339/Put/seqid=0 2024-12-08T00:14:40,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741937_1113 (size=56733) 2024-12-08T00:14:40,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/92c54d1427ef4eeea8d587f3732d30f7 2024-12-08T00:14:40,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T00:14:40,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:40,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:40,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:40,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:40,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:40,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/98c8a479affd4ff5ab9801372e6eaa00 is 50, key is test_row_0/B:col10/1733616880339/Put/seqid=0 2024-12-08T00:14:40,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741938_1114 (size=12001) 2024-12-08T00:14:40,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/98c8a479affd4ff5ab9801372e6eaa00 2024-12-08T00:14:40,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616940521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616940521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616940521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616940521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616940531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/5a32b0a024c04401bdda190765989b00 is 50, key is test_row_0/C:col10/1733616880339/Put/seqid=0 2024-12-08T00:14:40,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741939_1115 (size=12001) 2024-12-08T00:14:40,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/5a32b0a024c04401bdda190765989b00 2024-12-08T00:14:40,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/92c54d1427ef4eeea8d587f3732d30f7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/92c54d1427ef4eeea8d587f3732d30f7 2024-12-08T00:14:40,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/92c54d1427ef4eeea8d587f3732d30f7, entries=300, sequenceid=14, filesize=55.4 K 2024-12-08T00:14:40,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/98c8a479affd4ff5ab9801372e6eaa00 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/98c8a479affd4ff5ab9801372e6eaa00 2024-12-08T00:14:40,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/98c8a479affd4ff5ab9801372e6eaa00, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T00:14:40,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/5a32b0a024c04401bdda190765989b00 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5a32b0a024c04401bdda190765989b00 2024-12-08T00:14:40,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5a32b0a024c04401bdda190765989b00, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T00:14:40,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for c31d593296c0be255698232087c08858 in 278ms, sequenceid=14, compaction requested=false 2024-12-08T00:14:40,619 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T00:14:40,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:40,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T00:14:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:40,637 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=147.60 KB heapSize=377.95 KB 2024-12-08T00:14:40,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:40,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:40,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:40,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:40,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:40,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T00:14:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084d6f42c10f2940b4bfa8eff351bf9bbb_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616880404/Put/seqid=0 2024-12-08T00:14:40,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741940_1116 (size=12154) 2024-12-08T00:14:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:40,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616940740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616940741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616940745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616940747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616940748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616940851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616940851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616940861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616940861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T00:14:41,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616941053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616941061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616941061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:41,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616941071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616941071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084d6f42c10f2940b4bfa8eff351bf9bbb_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084d6f42c10f2940b4bfa8eff351bf9bbb_c31d593296c0be255698232087c08858 2024-12-08T00:14:41,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/91f4262913f74147a7829566621732e8, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:41,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/91f4262913f74147a7829566621732e8 is 175, key is test_row_0/A:col10/1733616880404/Put/seqid=0 2024-12-08T00:14:41,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741941_1117 (size=30955) 2024-12-08T00:14:41,098 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=43, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/91f4262913f74147a7829566621732e8 2024-12-08T00:14:41,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4d97756d30ba41dfa1348db000eebf05 is 50, key is test_row_0/B:col10/1733616880404/Put/seqid=0 2024-12-08T00:14:41,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741942_1118 (size=12001) 2024-12-08T00:14:41,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616941371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616941371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616941381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616941382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T00:14:41,539 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4d97756d30ba41dfa1348db000eebf05 2024-12-08T00:14:41,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/642798b76461400b853fb58fe5a82eb1 is 50, key is test_row_0/C:col10/1733616880404/Put/seqid=0 2024-12-08T00:14:41,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741943_1119 (size=12001) 2024-12-08T00:14:41,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616941561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,567 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/642798b76461400b853fb58fe5a82eb1 2024-12-08T00:14:41,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/91f4262913f74147a7829566621732e8 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/91f4262913f74147a7829566621732e8 2024-12-08T00:14:41,583 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/91f4262913f74147a7829566621732e8, entries=150, sequenceid=43, filesize=30.2 K 2024-12-08T00:14:41,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4d97756d30ba41dfa1348db000eebf05 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4d97756d30ba41dfa1348db000eebf05 2024-12-08T00:14:41,595 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4d97756d30ba41dfa1348db000eebf05, entries=150, sequenceid=43, filesize=11.7 K 2024-12-08T00:14:41,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/642798b76461400b853fb58fe5a82eb1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/642798b76461400b853fb58fe5a82eb1 2024-12-08T00:14:41,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/642798b76461400b853fb58fe5a82eb1, entries=150, sequenceid=43, filesize=11.7 K 2024-12-08T00:14:41,608 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~377.91 KB/386976, currentSize=46.96 KB/48090 for c31d593296c0be255698232087c08858 in 970ms, sequenceid=43, compaction requested=false 2024-12-08T00:14:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-08T00:14:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-08T00:14:41,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-08T00:14:41,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2810 sec 2024-12-08T00:14:41,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.2900 sec 2024-12-08T00:14:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:41,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T00:14:41,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:41,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:41,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:41,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:41,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:41,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:41,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089570ea7ec0344c4eb07b3dd285b3ab25_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616881882/Put/seqid=0 2024-12-08T00:14:41,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741944_1120 (size=17034) 2024-12-08T00:14:41,922 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:41,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089570ea7ec0344c4eb07b3dd285b3ab25_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089570ea7ec0344c4eb07b3dd285b3ab25_c31d593296c0be255698232087c08858 2024-12-08T00:14:41,928 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/e4c80853a78b4fc28a08e5e7c430b23d, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:41,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/e4c80853a78b4fc28a08e5e7c430b23d is 175, key is test_row_0/A:col10/1733616881882/Put/seqid=0 2024-12-08T00:14:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616941930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616941931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616941934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616941936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741945_1121 (size=48139) 2024-12-08T00:14:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616942041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616942041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616942051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616942051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616942251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616942251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616942260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616942260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,294 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T00:14:42,348 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/e4c80853a78b4fc28a08e5e7c430b23d 2024-12-08T00:14:42,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/0e4c25c5367a44d4b3cd6bf95b3458cc is 50, key is test_row_0/B:col10/1733616881882/Put/seqid=0 2024-12-08T00:14:42,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741946_1122 (size=12001) 2024-12-08T00:14:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T00:14:42,471 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:42,472 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-08T00:14:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T00:14:42,474 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:42,475 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:42,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616942561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616942561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616942571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616942571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T00:14:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616942580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:42,584 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in c31d593296c0be255698232087c08858 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:42,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T00:14:42,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:42,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:42,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:42,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:42,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:42,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/0e4c25c5367a44d4b3cd6bf95b3458cc 2024-12-08T00:14:42,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/5923f006a6244407bcd9b7c226eb953a is 50, key is test_row_0/C:col10/1733616881882/Put/seqid=0 2024-12-08T00:14:42,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741947_1123 (size=12001) 2024-12-08T00:14:42,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T00:14:42,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:42,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T00:14:42,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T00:14:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:42,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:42,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:43,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616943070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:43,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616943070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616943081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616943081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:43,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T00:14:43,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:43,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:43,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:43,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:43,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T00:14:43,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/5923f006a6244407bcd9b7c226eb953a 2024-12-08T00:14:43,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/e4c80853a78b4fc28a08e5e7c430b23d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/e4c80853a78b4fc28a08e5e7c430b23d 2024-12-08T00:14:43,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/e4c80853a78b4fc28a08e5e7c430b23d, entries=250, sequenceid=55, filesize=47.0 K 2024-12-08T00:14:43,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/0e4c25c5367a44d4b3cd6bf95b3458cc as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0e4c25c5367a44d4b3cd6bf95b3458cc 2024-12-08T00:14:43,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0e4c25c5367a44d4b3cd6bf95b3458cc, entries=150, sequenceid=55, filesize=11.7 K 2024-12-08T00:14:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/5923f006a6244407bcd9b7c226eb953a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5923f006a6244407bcd9b7c226eb953a 2024-12-08T00:14:43,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5923f006a6244407bcd9b7c226eb953a, entries=150, sequenceid=55, filesize=11.7 K 2024-12-08T00:14:43,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c31d593296c0be255698232087c08858 in 1318ms, sequenceid=55, compaction requested=true 2024-12-08T00:14:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:43,204 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:43,204 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:43,205 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:43,206 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/B is initiating minor compaction (all files) 2024-12-08T00:14:43,206 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/B in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:43,206 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/98c8a479affd4ff5ab9801372e6eaa00, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4d97756d30ba41dfa1348db000eebf05, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0e4c25c5367a44d4b3cd6bf95b3458cc] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=35.2 K 2024-12-08T00:14:43,206 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:43,206 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/A is initiating minor compaction (all files) 2024-12-08T00:14:43,206 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/A in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:43,206 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c8a479affd4ff5ab9801372e6eaa00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733616880332 2024-12-08T00:14:43,206 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/92c54d1427ef4eeea8d587f3732d30f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/91f4262913f74147a7829566621732e8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/e4c80853a78b4fc28a08e5e7c430b23d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=132.6 K 2024-12-08T00:14:43,206 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:43,207 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d97756d30ba41dfa1348db000eebf05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733616880400 2024-12-08T00:14:43,207 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/92c54d1427ef4eeea8d587f3732d30f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/91f4262913f74147a7829566621732e8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/e4c80853a78b4fc28a08e5e7c430b23d] 2024-12-08T00:14:43,207 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e4c25c5367a44d4b3cd6bf95b3458cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733616880735 2024-12-08T00:14:43,208 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c54d1427ef4eeea8d587f3732d30f7, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733616880331 2024-12-08T00:14:43,209 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91f4262913f74147a7829566621732e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733616880400 2024-12-08T00:14:43,209 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c80853a78b4fc28a08e5e7c430b23d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733616880735 2024-12-08T00:14:43,217 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:43,218 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/cad446863c044f3a98fbf88dadec1094 is 50, key is test_row_0/B:col10/1733616881882/Put/seqid=0 2024-12-08T00:14:43,225 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:43,232 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412087b0dce20885b4833868ad1a64e24f5ef_c31d593296c0be255698232087c08858 store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:43,236 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412087b0dce20885b4833868ad1a64e24f5ef_c31d593296c0be255698232087c08858, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:43,236 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087b0dce20885b4833868ad1a64e24f5ef_c31d593296c0be255698232087c08858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:43,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T00:14:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:43,240 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:14:43,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:43,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:43,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:43,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:43,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:43,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:43,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741949_1125 (size=4469) 2024-12-08T00:14:43,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741948_1124 (size=12104) 2024-12-08T00:14:43,258 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#A#compaction#106 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:43,259 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/c5c4d0335f5f46258d5613c55e8a8d7a is 175, key is test_row_0/A:col10/1733616881882/Put/seqid=0 2024-12-08T00:14:43,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bd99caf9060b4163961d50621dacebf0_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616881934/Put/seqid=0 2024-12-08T00:14:43,267 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/cad446863c044f3a98fbf88dadec1094 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/cad446863c044f3a98fbf88dadec1094 2024-12-08T00:14:43,273 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/B of c31d593296c0be255698232087c08858 into cad446863c044f3a98fbf88dadec1094(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:43,274 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:43,274 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/B, priority=13, startTime=1733616883204; duration=0sec 2024-12-08T00:14:43,274 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:43,274 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:B 2024-12-08T00:14:43,274 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:43,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741950_1126 (size=31058) 2024-12-08T00:14:43,276 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:43,276 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/C is initiating minor compaction (all files) 2024-12-08T00:14:43,276 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/C in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:43,276 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5a32b0a024c04401bdda190765989b00, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/642798b76461400b853fb58fe5a82eb1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5923f006a6244407bcd9b7c226eb953a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=35.2 K 2024-12-08T00:14:43,276 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a32b0a024c04401bdda190765989b00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733616880332 2024-12-08T00:14:43,277 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 642798b76461400b853fb58fe5a82eb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733616880400 2024-12-08T00:14:43,277 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 5923f006a6244407bcd9b7c226eb953a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733616880735 2024-12-08T00:14:43,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741951_1127 (size=12154) 2024-12-08T00:14:43,290 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#C#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:43,291 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/8fb958bcc8df4a058189fa730ffc3561 is 50, key is test_row_0/C:col10/1733616881882/Put/seqid=0 2024-12-08T00:14:43,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741952_1128 (size=12104) 2024-12-08T00:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T00:14:43,681 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/c5c4d0335f5f46258d5613c55e8a8d7a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/c5c4d0335f5f46258d5613c55e8a8d7a 2024-12-08T00:14:43,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:43,689 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/A of c31d593296c0be255698232087c08858 into c5c4d0335f5f46258d5613c55e8a8d7a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:43,689 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:43,689 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/A, priority=13, startTime=1733616883204; duration=0sec 2024-12-08T00:14:43,689 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:43,689 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:A 2024-12-08T00:14:43,690 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bd99caf9060b4163961d50621dacebf0_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bd99caf9060b4163961d50621dacebf0_c31d593296c0be255698232087c08858 2024-12-08T00:14:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/87f811767e434925aa8d703680a8af22, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:43,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/87f811767e434925aa8d703680a8af22 is 175, key is test_row_0/A:col10/1733616881934/Put/seqid=0 2024-12-08T00:14:43,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741953_1129 (size=30955) 2024-12-08T00:14:43,714 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/8fb958bcc8df4a058189fa730ffc3561 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8fb958bcc8df4a058189fa730ffc3561 2024-12-08T00:14:43,720 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/C of c31d593296c0be255698232087c08858 into 8fb958bcc8df4a058189fa730ffc3561(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:43,720 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:43,720 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/C, priority=13, startTime=1733616883204; duration=0sec 2024-12-08T00:14:43,721 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:43,721 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:C 2024-12-08T00:14:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:44,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:44,098 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=75, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/87f811767e434925aa8d703680a8af22 2024-12-08T00:14:44,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616944095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616944096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616944098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616944098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4fd980b1c05b47a2b4204f3c77287489 is 50, key is test_row_0/B:col10/1733616881934/Put/seqid=0 2024-12-08T00:14:44,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741954_1130 (size=12001) 2024-12-08T00:14:44,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616944211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616944211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616944211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616944211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616944421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616944421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616944421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616944421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,513 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4fd980b1c05b47a2b4204f3c77287489 2024-12-08T00:14:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/43aec6df38864ccf800ebd4e8034423d is 50, key is test_row_0/C:col10/1733616881934/Put/seqid=0 2024-12-08T00:14:44,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741955_1131 (size=12001) 2024-12-08T00:14:44,537 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/43aec6df38864ccf800ebd4e8034423d 2024-12-08T00:14:44,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/87f811767e434925aa8d703680a8af22 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/87f811767e434925aa8d703680a8af22 2024-12-08T00:14:44,553 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/87f811767e434925aa8d703680a8af22, entries=150, sequenceid=75, filesize=30.2 K 2024-12-08T00:14:44,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4fd980b1c05b47a2b4204f3c77287489 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4fd980b1c05b47a2b4204f3c77287489 2024-12-08T00:14:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,561 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4fd980b1c05b47a2b4204f3c77287489, entries=150, sequenceid=75, filesize=11.7 K 2024-12-08T00:14:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/43aec6df38864ccf800ebd4e8034423d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/43aec6df38864ccf800ebd4e8034423d 2024-12-08T00:14:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,567 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/43aec6df38864ccf800ebd4e8034423d, entries=150, sequenceid=75, filesize=11.7 K 2024-12-08T00:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,569 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c31d593296c0be255698232087c08858 in 1330ms, sequenceid=75, compaction requested=false 2024-12-08T00:14:44,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:44,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:44,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-08T00:14:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-08T00:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-08T00:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0960 sec 2024-12-08T00:14:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.1030 sec 2024-12-08T00:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:44,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T00:14:44,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:44,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:44,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:44,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:44,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:44,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T00:14:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,623 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:44,623 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-08T00:14:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T00:14:44,626 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c931e6cc486145cea81ce15624fc24d7_c31d593296c0be255698232087c08858 is 50, key is test_row_1/A:col10/1733616884612/Put/seqid=0 2024-12-08T00:14:44,628 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:44,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741956_1132 (size=9714) 2024-12-08T00:14:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T00:14:44,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616944740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616944740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616944742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616944742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616944743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T00:14:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:44,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:44,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616944851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616944851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616944861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616944861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:44,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616944861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:44,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:44,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T00:14:44,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:44,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:44,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:44,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:44,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T00:14:45,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:45,040 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c931e6cc486145cea81ce15624fc24d7_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c931e6cc486145cea81ce15624fc24d7_c31d593296c0be255698232087c08858 2024-12-08T00:14:45,041 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/1d95e4cb373f49439deb7f6e8ecbc6ea, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:45,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/1d95e4cb373f49439deb7f6e8ecbc6ea is 175, key is test_row_1/A:col10/1733616884612/Put/seqid=0 2024-12-08T00:14:45,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741957_1133 (size=22361) 2024-12-08T00:14:45,046 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/1d95e4cb373f49439deb7f6e8ecbc6ea 2024-12-08T00:14:45,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/66c66656c35c4a63aecac591471c78a0 is 50, key is test_row_1/B:col10/1733616884612/Put/seqid=0 2024-12-08T00:14:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616945061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616945061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741958_1134 (size=9657) 2024-12-08T00:14:45,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616945071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616945071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616945071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T00:14:45,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T00:14:45,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:45,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T00:14:45,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616945371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616945371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616945381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616945381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616945381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T00:14:45,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:45,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/66c66656c35c4a63aecac591471c78a0 2024-12-08T00:14:45,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/c4812367ceb34dd2bc3fb1e29ea4bf28 is 50, key is test_row_1/C:col10/1733616884612/Put/seqid=0 2024-12-08T00:14:45,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741959_1135 (size=9657) 2024-12-08T00:14:45,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T00:14:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T00:14:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T00:14:45,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T00:14:45,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:45,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:45,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/c4812367ceb34dd2bc3fb1e29ea4bf28 2024-12-08T00:14:45,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616945881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616945881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/1d95e4cb373f49439deb7f6e8ecbc6ea as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1d95e4cb373f49439deb7f6e8ecbc6ea 2024-12-08T00:14:45,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616945891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616945891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616945891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1d95e4cb373f49439deb7f6e8ecbc6ea, entries=100, sequenceid=92, filesize=21.8 K 2024-12-08T00:14:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/66c66656c35c4a63aecac591471c78a0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66c66656c35c4a63aecac591471c78a0 2024-12-08T00:14:45,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66c66656c35c4a63aecac591471c78a0, entries=100, sequenceid=92, filesize=9.4 K 2024-12-08T00:14:45,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/c4812367ceb34dd2bc3fb1e29ea4bf28 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c4812367ceb34dd2bc3fb1e29ea4bf28 2024-12-08T00:14:45,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c4812367ceb34dd2bc3fb1e29ea4bf28, entries=100, sequenceid=92, filesize=9.4 K 2024-12-08T00:14:45,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c31d593296c0be255698232087c08858 in 1292ms, sequenceid=92, compaction requested=true 2024-12-08T00:14:45,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:45,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:45,906 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:45,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:45,906 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:45,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:45,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:45,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:45,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:45,907 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:45,907 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84374 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:45,907 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/A is initiating minor compaction (all files) 2024-12-08T00:14:45,907 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/B is initiating minor compaction (all files) 2024-12-08T00:14:45,907 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/B in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,908 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/cad446863c044f3a98fbf88dadec1094, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4fd980b1c05b47a2b4204f3c77287489, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66c66656c35c4a63aecac591471c78a0] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=33.0 K 2024-12-08T00:14:45,908 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/A in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,908 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/c5c4d0335f5f46258d5613c55e8a8d7a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/87f811767e434925aa8d703680a8af22, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1d95e4cb373f49439deb7f6e8ecbc6ea] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=82.4 K 2024-12-08T00:14:45,908 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:45,908 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/c5c4d0335f5f46258d5613c55e8a8d7a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/87f811767e434925aa8d703680a8af22, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1d95e4cb373f49439deb7f6e8ecbc6ea] 2024-12-08T00:14:45,908 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting cad446863c044f3a98fbf88dadec1094, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733616880735 2024-12-08T00:14:45,909 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd980b1c05b47a2b4204f3c77287489, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733616881924 2024-12-08T00:14:45,909 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c4d0335f5f46258d5613c55e8a8d7a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733616880735 2024-12-08T00:14:45,909 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 66c66656c35c4a63aecac591471c78a0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733616884096 2024-12-08T00:14:45,909 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87f811767e434925aa8d703680a8af22, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733616881924 2024-12-08T00:14:45,910 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d95e4cb373f49439deb7f6e8ecbc6ea, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733616884096 2024-12-08T00:14:45,917 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:45,918 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/0ae2f2f5792d4910a03b11eee708f02f is 50, key is test_row_0/B:col10/1733616881934/Put/seqid=0 2024-12-08T00:14:45,921 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:45,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741960_1136 (size=12207) 2024-12-08T00:14:45,933 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120892114c8bd971436992d6a4ef068cf8ba_c31d593296c0be255698232087c08858 store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:45,935 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120892114c8bd971436992d6a4ef068cf8ba_c31d593296c0be255698232087c08858, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:45,935 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120892114c8bd971436992d6a4ef068cf8ba_c31d593296c0be255698232087c08858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:45,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741961_1137 (size=4469) 2024-12-08T00:14:46,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T00:14:46,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:46,007 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T00:14:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:46,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f47cd5efe7d8476c8549cac3e1209562_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616884738/Put/seqid=0 2024-12-08T00:14:46,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741962_1138 (size=12154) 2024-12-08T00:14:46,339 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/0ae2f2f5792d4910a03b11eee708f02f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0ae2f2f5792d4910a03b11eee708f02f 2024-12-08T00:14:46,344 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/B of c31d593296c0be255698232087c08858 into 0ae2f2f5792d4910a03b11eee708f02f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:46,345 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:46,345 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/B, priority=13, startTime=1733616885906; duration=0sec 2024-12-08T00:14:46,345 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:46,345 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:B 2024-12-08T00:14:46,345 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:46,346 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:46,346 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/C is initiating minor compaction (all files) 2024-12-08T00:14:46,346 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/C in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:46,346 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8fb958bcc8df4a058189fa730ffc3561, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/43aec6df38864ccf800ebd4e8034423d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c4812367ceb34dd2bc3fb1e29ea4bf28] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=33.0 K 2024-12-08T00:14:46,346 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb958bcc8df4a058189fa730ffc3561, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733616880735 2024-12-08T00:14:46,347 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 43aec6df38864ccf800ebd4e8034423d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733616881924 2024-12-08T00:14:46,347 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c4812367ceb34dd2bc3fb1e29ea4bf28, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733616884096 2024-12-08T00:14:46,352 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#A#compaction#115 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:46,353 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/1f28b4a43ab94fc6b9c44c508cbf4645 is 175, key is test_row_0/A:col10/1733616881934/Put/seqid=0 2024-12-08T00:14:46,355 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:46,355 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/2f19a508cc7a4e3e9382eab086ac5c2f is 50, key is test_row_0/C:col10/1733616881934/Put/seqid=0 2024-12-08T00:14:46,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741963_1139 (size=31268) 2024-12-08T00:14:46,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741964_1140 (size=12207) 2024-12-08T00:14:46,398 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/1f28b4a43ab94fc6b9c44c508cbf4645 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1f28b4a43ab94fc6b9c44c508cbf4645 2024-12-08T00:14:46,405 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/A of c31d593296c0be255698232087c08858 into 1f28b4a43ab94fc6b9c44c508cbf4645(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:46,405 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:46,405 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/A, priority=13, startTime=1733616885906; duration=0sec 2024-12-08T00:14:46,405 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:46,405 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:A 2024-12-08T00:14:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:46,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f47cd5efe7d8476c8549cac3e1209562_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f47cd5efe7d8476c8549cac3e1209562_c31d593296c0be255698232087c08858 2024-12-08T00:14:46,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/593c14b56b8e4c2b90b8bc1161c3d482, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/593c14b56b8e4c2b90b8bc1161c3d482 is 175, key is test_row_0/A:col10/1733616884738/Put/seqid=0 2024-12-08T00:14:46,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741965_1141 (size=30955) 2024-12-08T00:14:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T00:14:46,777 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/2f19a508cc7a4e3e9382eab086ac5c2f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/2f19a508cc7a4e3e9382eab086ac5c2f 2024-12-08T00:14:46,786 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/C of c31d593296c0be255698232087c08858 into 2f19a508cc7a4e3e9382eab086ac5c2f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:46,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:46,787 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/C, priority=13, startTime=1733616885907; duration=0sec 2024-12-08T00:14:46,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:46,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:C 2024-12-08T00:14:46,830 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/593c14b56b8e4c2b90b8bc1161c3d482 2024-12-08T00:14:46,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/c124d7d9402747dd96d7e51364627067 is 50, key is test_row_0/B:col10/1733616884738/Put/seqid=0 2024-12-08T00:14:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:46,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:46,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741966_1142 (size=12001) 2024-12-08T00:14:46,899 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/c124d7d9402747dd96d7e51364627067 2024-12-08T00:14:46,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/d4db4a1367744fc4a2f0a41fcfd1a9b6 is 50, key is test_row_0/C:col10/1733616884738/Put/seqid=0 2024-12-08T00:14:46,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616946915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616946918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616946915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616946919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616946925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:46,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741967_1143 (size=12001) 2024-12-08T00:14:46,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/d4db4a1367744fc4a2f0a41fcfd1a9b6 2024-12-08T00:14:46,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/593c14b56b8e4c2b90b8bc1161c3d482 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/593c14b56b8e4c2b90b8bc1161c3d482 2024-12-08T00:14:46,992 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/593c14b56b8e4c2b90b8bc1161c3d482, entries=150, sequenceid=111, filesize=30.2 K 2024-12-08T00:14:46,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/c124d7d9402747dd96d7e51364627067 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c124d7d9402747dd96d7e51364627067 2024-12-08T00:14:46,999 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c124d7d9402747dd96d7e51364627067, entries=150, sequenceid=111, filesize=11.7 K 2024-12-08T00:14:47,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/d4db4a1367744fc4a2f0a41fcfd1a9b6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/d4db4a1367744fc4a2f0a41fcfd1a9b6 2024-12-08T00:14:47,016 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/d4db4a1367744fc4a2f0a41fcfd1a9b6, entries=150, sequenceid=111, filesize=11.7 K 2024-12-08T00:14:47,018 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c31d593296c0be255698232087c08858 in 1011ms, sequenceid=111, compaction requested=false 2024-12-08T00:14:47,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:47,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:47,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-08T00:14:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-08T00:14:47,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-08T00:14:47,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3930 sec 2024-12-08T00:14:47,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4020 sec 2024-12-08T00:14:47,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T00:14:47,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:47,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:47,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:47,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:47,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:47,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:47,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083de4b52427ce45ef8e35138ad13568d9_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616887033/Put/seqid=0 2024-12-08T00:14:47,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741968_1144 (size=14594) 2024-12-08T00:14:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616947093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616947095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616947097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616947097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616947098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616947211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616947211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616947211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616947211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616947212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616947420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616947420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616947421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616947421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616947421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,471 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:47,478 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083de4b52427ce45ef8e35138ad13568d9_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083de4b52427ce45ef8e35138ad13568d9_c31d593296c0be255698232087c08858 2024-12-08T00:14:47,479 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/27f3b8e06c004c18b9b09e03114c6b95, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:47,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/27f3b8e06c004c18b9b09e03114c6b95 is 175, key is test_row_0/A:col10/1733616887033/Put/seqid=0 2024-12-08T00:14:47,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741969_1145 (size=39549) 2024-12-08T00:14:47,575 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T00:14:47,575 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T00:14:47,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616947731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616947731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616947731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616947731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616947732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:47,889 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/27f3b8e06c004c18b9b09e03114c6b95 2024-12-08T00:14:47,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/71c8ce5023464fafb0328cd9db58dae5 is 50, key is test_row_0/B:col10/1733616887033/Put/seqid=0 2024-12-08T00:14:47,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741970_1146 (size=12001) 2024-12-08T00:14:48,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616948241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:48,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616948241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:48,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616948241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:48,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616948241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:48,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616948251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:48,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/71c8ce5023464fafb0328cd9db58dae5 2024-12-08T00:14:48,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/cc2a619cf45a4c07945f9b223cd4238c is 50, key is test_row_0/C:col10/1733616887033/Put/seqid=0 2024-12-08T00:14:48,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741971_1147 (size=12001) 2024-12-08T00:14:48,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/cc2a619cf45a4c07945f9b223cd4238c 2024-12-08T00:14:48,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/27f3b8e06c004c18b9b09e03114c6b95 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/27f3b8e06c004c18b9b09e03114c6b95 2024-12-08T00:14:48,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/27f3b8e06c004c18b9b09e03114c6b95, entries=200, sequenceid=128, filesize=38.6 K 2024-12-08T00:14:48,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/71c8ce5023464fafb0328cd9db58dae5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/71c8ce5023464fafb0328cd9db58dae5 2024-12-08T00:14:48,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/71c8ce5023464fafb0328cd9db58dae5, entries=150, sequenceid=128, filesize=11.7 K 2024-12-08T00:14:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/cc2a619cf45a4c07945f9b223cd4238c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/cc2a619cf45a4c07945f9b223cd4238c 2024-12-08T00:14:48,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/cc2a619cf45a4c07945f9b223cd4238c, entries=150, sequenceid=128, filesize=11.7 K 2024-12-08T00:14:48,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c31d593296c0be255698232087c08858 in 1702ms, sequenceid=128, compaction requested=true 2024-12-08T00:14:48,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:48,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:48,738 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:48,738 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:48,739 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:48,739 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101772 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:48,739 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/B is initiating minor compaction (all files) 2024-12-08T00:14:48,739 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/A is initiating minor compaction (all files) 2024-12-08T00:14:48,739 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/B in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:48,739 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/A in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:48,739 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1f28b4a43ab94fc6b9c44c508cbf4645, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/593c14b56b8e4c2b90b8bc1161c3d482, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/27f3b8e06c004c18b9b09e03114c6b95] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=99.4 K 2024-12-08T00:14:48,739 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0ae2f2f5792d4910a03b11eee708f02f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c124d7d9402747dd96d7e51364627067, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/71c8ce5023464fafb0328cd9db58dae5] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=35.4 K 2024-12-08T00:14:48,739 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:48,739 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1f28b4a43ab94fc6b9c44c508cbf4645, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/593c14b56b8e4c2b90b8bc1161c3d482, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/27f3b8e06c004c18b9b09e03114c6b95] 2024-12-08T00:14:48,740 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae2f2f5792d4910a03b11eee708f02f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733616881934 2024-12-08T00:14:48,740 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f28b4a43ab94fc6b9c44c508cbf4645, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733616881934 2024-12-08T00:14:48,740 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c124d7d9402747dd96d7e51364627067, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733616884738 2024-12-08T00:14:48,740 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 593c14b56b8e4c2b90b8bc1161c3d482, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733616884738 2024-12-08T00:14:48,740 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c8ce5023464fafb0328cd9db58dae5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616886911 2024-12-08T00:14:48,740 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27f3b8e06c004c18b9b09e03114c6b95, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616886909 2024-12-08T00:14:48,748 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:48,749 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:48,749 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/bfc16206580b4b8d9b9d5ea43e6cf920 is 50, key is test_row_0/B:col10/1733616887033/Put/seqid=0 2024-12-08T00:14:48,751 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120816ce7413947647cbb7859e573dc537ef_c31d593296c0be255698232087c08858 store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:48,753 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120816ce7413947647cbb7859e573dc537ef_c31d593296c0be255698232087c08858, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:48,753 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120816ce7413947647cbb7859e573dc537ef_c31d593296c0be255698232087c08858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:48,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741973_1149 (size=4469) 2024-12-08T00:14:48,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741972_1148 (size=12309) 2024-12-08T00:14:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T00:14:48,801 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:48,802 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-08T00:14:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T00:14:48,804 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:48,805 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:48,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T00:14:48,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T00:14:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:48,957 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T00:14:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:48,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a61c769d4825429cb4cb22baa6e947e7_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616887061/Put/seqid=0 2024-12-08T00:14:48,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741974_1150 (size=12304) 2024-12-08T00:14:48,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:48,973 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a61c769d4825429cb4cb22baa6e947e7_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a61c769d4825429cb4cb22baa6e947e7_c31d593296c0be255698232087c08858 2024-12-08T00:14:48,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/464c14413fec4381aa5bebdedc317d4a, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:48,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/464c14413fec4381aa5bebdedc317d4a is 175, key is test_row_0/A:col10/1733616887061/Put/seqid=0 2024-12-08T00:14:48,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741975_1151 (size=31105) 2024-12-08T00:14:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T00:14:49,167 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#A#compaction#124 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:49,168 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/080717c38dd54ae0af98189c2f1aa55e is 175, key is test_row_0/A:col10/1733616887033/Put/seqid=0 2024-12-08T00:14:49,176 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/bfc16206580b4b8d9b9d5ea43e6cf920 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/bfc16206580b4b8d9b9d5ea43e6cf920 2024-12-08T00:14:49,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741976_1152 (size=31263) 2024-12-08T00:14:49,181 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/B of c31d593296c0be255698232087c08858 into bfc16206580b4b8d9b9d5ea43e6cf920(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:49,181 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:49,181 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/B, priority=13, startTime=1733616888738; duration=0sec 2024-12-08T00:14:49,181 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:49,181 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:B 2024-12-08T00:14:49,181 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:49,182 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:49,182 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/C is initiating minor compaction (all files) 2024-12-08T00:14:49,182 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/C in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:49,182 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/2f19a508cc7a4e3e9382eab086ac5c2f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/d4db4a1367744fc4a2f0a41fcfd1a9b6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/cc2a619cf45a4c07945f9b223cd4238c] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=35.4 K 2024-12-08T00:14:49,183 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f19a508cc7a4e3e9382eab086ac5c2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733616881934 2024-12-08T00:14:49,183 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d4db4a1367744fc4a2f0a41fcfd1a9b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733616884738 2024-12-08T00:14:49,183 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting cc2a619cf45a4c07945f9b223cd4238c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616886911 2024-12-08T00:14:49,189 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:49,189 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/c541287fd67e430394fea62a859075a5 is 50, key is test_row_0/C:col10/1733616887033/Put/seqid=0 2024-12-08T00:14:49,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741977_1153 (size=12309) 2024-12-08T00:14:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:49,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:49,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616949261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616949266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616949268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616949269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616949269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,381 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/464c14413fec4381aa5bebdedc317d4a 2024-12-08T00:14:49,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616949380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616949381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616949381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616949381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616949381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/65c308356cf540edbfce719d409a9c06 is 50, key is test_row_0/B:col10/1733616887061/Put/seqid=0 2024-12-08T00:14:49,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741978_1154 (size=12151) 2024-12-08T00:14:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T00:14:49,578 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/080717c38dd54ae0af98189c2f1aa55e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/080717c38dd54ae0af98189c2f1aa55e 2024-12-08T00:14:49,582 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/A of c31d593296c0be255698232087c08858 into 080717c38dd54ae0af98189c2f1aa55e(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:49,582 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:49,582 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/A, priority=13, startTime=1733616888737; duration=0sec 2024-12-08T00:14:49,583 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:49,583 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:A 2024-12-08T00:14:49,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616949590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616949590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616949590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616949591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616949591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,598 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/c541287fd67e430394fea62a859075a5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c541287fd67e430394fea62a859075a5 2024-12-08T00:14:49,603 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/C of c31d593296c0be255698232087c08858 into c541287fd67e430394fea62a859075a5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:49,603 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:49,603 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/C, priority=13, startTime=1733616888738; duration=0sec 2024-12-08T00:14:49,603 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:49,603 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:C 2024-12-08T00:14:49,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/65c308356cf540edbfce719d409a9c06 2024-12-08T00:14:49,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/63702917290542aeba0cd5c32ad15030 is 50, key is test_row_0/C:col10/1733616887061/Put/seqid=0 2024-12-08T00:14:49,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741979_1155 (size=12151) 2024-12-08T00:14:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616949901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616949901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616949901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616949901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616949901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T00:14:50,206 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/63702917290542aeba0cd5c32ad15030 2024-12-08T00:14:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/464c14413fec4381aa5bebdedc317d4a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/464c14413fec4381aa5bebdedc317d4a 2024-12-08T00:14:50,215 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/464c14413fec4381aa5bebdedc317d4a, entries=150, sequenceid=147, filesize=30.4 K 2024-12-08T00:14:50,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/65c308356cf540edbfce719d409a9c06 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65c308356cf540edbfce719d409a9c06 2024-12-08T00:14:50,221 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65c308356cf540edbfce719d409a9c06, entries=150, sequenceid=147, filesize=11.9 K 2024-12-08T00:14:50,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/63702917290542aeba0cd5c32ad15030 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/63702917290542aeba0cd5c32ad15030 2024-12-08T00:14:50,229 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/63702917290542aeba0cd5c32ad15030, entries=150, sequenceid=147, filesize=11.9 K 2024-12-08T00:14:50,230 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c31d593296c0be255698232087c08858 in 1273ms, sequenceid=147, compaction requested=false 2024-12-08T00:14:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-08T00:14:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-08T00:14:50,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-08T00:14:50,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-12-08T00:14:50,236 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4330 sec 2024-12-08T00:14:50,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:50,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T00:14:50,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:50,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:50,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:50,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:50,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:50,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:50,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208635af24af94f4dee9c9ec573d18bfa8a_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616890413/Put/seqid=0 2024-12-08T00:14:50,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741980_1156 (size=17284) 2024-12-08T00:14:50,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:50,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616950426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,431 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208635af24af94f4dee9c9ec573d18bfa8a_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208635af24af94f4dee9c9ec573d18bfa8a_c31d593296c0be255698232087c08858 2024-12-08T00:14:50,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616950426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616950426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616950427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616950427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,432 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/f5a972700da34f959e724f4bd18de9d2, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:50,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/f5a972700da34f959e724f4bd18de9d2 is 175, key is test_row_0/A:col10/1733616890413/Put/seqid=0 2024-12-08T00:14:50,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741981_1157 (size=48389) 2024-12-08T00:14:50,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616950541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616950541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616950541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616950541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616950542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616950750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616950751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616950751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616950751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616950751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:50,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/f5a972700da34f959e724f4bd18de9d2 2024-12-08T00:14:50,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/b0694d4657b0475ebbc1d75880af079c is 50, key is test_row_0/B:col10/1733616890413/Put/seqid=0 2024-12-08T00:14:50,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741982_1158 (size=12151) 2024-12-08T00:14:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T00:14:50,951 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:50,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-08T00:14:50,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T00:14:50,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:50,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T00:14:51,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616951062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616951062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616951062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616951062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616951062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T00:14:51,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:51,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/b0694d4657b0475ebbc1d75880af079c 2024-12-08T00:14:51,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/ba757dbb280241118b10e6cc53577ab1 is 50, key is test_row_0/C:col10/1733616890413/Put/seqid=0 2024-12-08T00:14:51,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T00:14:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741983_1159 (size=12151) 2024-12-08T00:14:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T00:14:51,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T00:14:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,553 INFO [master/c1a8cf8d4cfa:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-08T00:14:51,553 INFO [master/c1a8cf8d4cfa:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-08T00:14:51,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T00:14:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:51,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616951571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616951571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616951571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616951571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616951571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T00:14:51,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/ba757dbb280241118b10e6cc53577ab1 2024-12-08T00:14:51,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/f5a972700da34f959e724f4bd18de9d2 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/f5a972700da34f959e724f4bd18de9d2 2024-12-08T00:14:51,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/f5a972700da34f959e724f4bd18de9d2, entries=250, sequenceid=169, filesize=47.3 K 2024-12-08T00:14:51,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/b0694d4657b0475ebbc1d75880af079c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b0694d4657b0475ebbc1d75880af079c 2024-12-08T00:14:51,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b0694d4657b0475ebbc1d75880af079c, entries=150, sequenceid=169, filesize=11.9 K 2024-12-08T00:14:51,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/ba757dbb280241118b10e6cc53577ab1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/ba757dbb280241118b10e6cc53577ab1 2024-12-08T00:14:51,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/ba757dbb280241118b10e6cc53577ab1, entries=150, sequenceid=169, filesize=11.9 K 2024-12-08T00:14:51,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c31d593296c0be255698232087c08858 in 1278ms, sequenceid=169, compaction requested=true 2024-12-08T00:14:51,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:51,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:51,691 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:51,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:51,691 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:51,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:51,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:51,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:51,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:51,694 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110757 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:51,694 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:51,694 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/A is initiating minor compaction (all files) 2024-12-08T00:14:51,694 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/B is initiating minor compaction (all files) 2024-12-08T00:14:51,694 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/A in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,694 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/B in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,694 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/bfc16206580b4b8d9b9d5ea43e6cf920, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65c308356cf540edbfce719d409a9c06, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b0694d4657b0475ebbc1d75880af079c] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=35.8 K 2024-12-08T00:14:51,694 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/080717c38dd54ae0af98189c2f1aa55e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/464c14413fec4381aa5bebdedc317d4a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/f5a972700da34f959e724f4bd18de9d2] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=108.2 K 2024-12-08T00:14:51,695 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,695 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/080717c38dd54ae0af98189c2f1aa55e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/464c14413fec4381aa5bebdedc317d4a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/f5a972700da34f959e724f4bd18de9d2] 2024-12-08T00:14:51,695 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc16206580b4b8d9b9d5ea43e6cf920, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616886911 2024-12-08T00:14:51,695 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 080717c38dd54ae0af98189c2f1aa55e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616886911 2024-12-08T00:14:51,695 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 65c308356cf540edbfce719d409a9c06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733616887058 2024-12-08T00:14:51,695 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 464c14413fec4381aa5bebdedc317d4a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733616887058 2024-12-08T00:14:51,695 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting b0694d4657b0475ebbc1d75880af079c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733616890412 2024-12-08T00:14:51,696 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a972700da34f959e724f4bd18de9d2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733616889267 2024-12-08T00:14:51,701 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:51,703 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:51,703 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/b5017e6bc19c48bd95eaddc9a35a17f7 is 50, key is test_row_0/B:col10/1733616890413/Put/seqid=0 2024-12-08T00:14:51,704 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412086e30c9ca4ed7423096d5df263d83cbae_c31d593296c0be255698232087c08858 store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:51,706 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412086e30c9ca4ed7423096d5df263d83cbae_c31d593296c0be255698232087c08858, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:51,706 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086e30c9ca4ed7423096d5df263d83cbae_c31d593296c0be255698232087c08858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:51,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T00:14:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:51,717 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:14:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:51,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741984_1160 (size=12561) 2024-12-08T00:14:51,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741985_1161 (size=4469) 2024-12-08T00:14:51,732 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#A#compaction#133 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:51,733 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/1fc3a862c8a644a2ae7ad33ea336c85e is 175, key is test_row_0/A:col10/1733616890413/Put/seqid=0 2024-12-08T00:14:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a44cfbdd43d5498186afc71c71309b23_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616890425/Put/seqid=0 2024-12-08T00:14:51,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741987_1163 (size=12304) 2024-12-08T00:14:51,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741986_1162 (size=31515) 2024-12-08T00:14:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T00:14:52,132 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/b5017e6bc19c48bd95eaddc9a35a17f7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b5017e6bc19c48bd95eaddc9a35a17f7 2024-12-08T00:14:52,137 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/B of c31d593296c0be255698232087c08858 into b5017e6bc19c48bd95eaddc9a35a17f7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:52,137 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:52,137 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/B, priority=13, startTime=1733616891691; duration=0sec 2024-12-08T00:14:52,138 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:52,138 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:B 2024-12-08T00:14:52,138 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:52,139 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:52,139 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/C is initiating minor compaction (all files) 2024-12-08T00:14:52,139 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/C in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:52,139 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c541287fd67e430394fea62a859075a5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/63702917290542aeba0cd5c32ad15030, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/ba757dbb280241118b10e6cc53577ab1] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=35.8 K 2024-12-08T00:14:52,139 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c541287fd67e430394fea62a859075a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616886911 2024-12-08T00:14:52,140 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 63702917290542aeba0cd5c32ad15030, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733616887058 2024-12-08T00:14:52,141 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ba757dbb280241118b10e6cc53577ab1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733616890412 2024-12-08T00:14:52,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:52,146 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a44cfbdd43d5498186afc71c71309b23_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a44cfbdd43d5498186afc71c71309b23_c31d593296c0be255698232087c08858 2024-12-08T00:14:52,146 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/1fc3a862c8a644a2ae7ad33ea336c85e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1fc3a862c8a644a2ae7ad33ea336c85e 2024-12-08T00:14:52,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/7c7d3c5ad1a542778001efb095ebd861, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:52,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/7c7d3c5ad1a542778001efb095ebd861 is 175, key is test_row_0/A:col10/1733616890425/Put/seqid=0 2024-12-08T00:14:52,151 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:52,151 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/e68abefea9ce47f8853498fd5ec75d60 is 50, key is test_row_0/C:col10/1733616890413/Put/seqid=0 2024-12-08T00:14:52,153 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/A of c31d593296c0be255698232087c08858 into 1fc3a862c8a644a2ae7ad33ea336c85e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:52,153 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:52,154 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/A, priority=13, startTime=1733616891691; duration=0sec 2024-12-08T00:14:52,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741988_1164 (size=31105) 2024-12-08T00:14:52,154 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:52,154 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:A 2024-12-08T00:14:52,157 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/7c7d3c5ad1a542778001efb095ebd861 2024-12-08T00:14:52,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741989_1165 (size=12561) 2024-12-08T00:14:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/c14f199b18f44cf7ae4cf9a370ebe737 is 50, key is test_row_0/B:col10/1733616890425/Put/seqid=0 2024-12-08T00:14:52,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741990_1166 (size=12151) 2024-12-08T00:14:52,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/e68abefea9ce47f8853498fd5ec75d60 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/e68abefea9ce47f8853498fd5ec75d60 2024-12-08T00:14:52,574 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/C of c31d593296c0be255698232087c08858 into e68abefea9ce47f8853498fd5ec75d60(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:52,574 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:52,574 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/C, priority=13, startTime=1733616891691; duration=0sec 2024-12-08T00:14:52,574 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:52,575 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:C 2024-12-08T00:14:52,576 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/c14f199b18f44cf7ae4cf9a370ebe737 2024-12-08T00:14:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/5ab07aa4bc994324b256afb95a15765e is 50, key is test_row_0/C:col10/1733616890425/Put/seqid=0 2024-12-08T00:14:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:52,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741991_1167 (size=12151) 2024-12-08T00:14:52,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616952599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616952599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616952600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616952601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616952601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616952710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616952710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616952711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616952711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616952711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616952920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616952921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616952921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616952921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616952921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:52,988 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/5ab07aa4bc994324b256afb95a15765e 2024-12-08T00:14:52,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/7c7d3c5ad1a542778001efb095ebd861 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/7c7d3c5ad1a542778001efb095ebd861 2024-12-08T00:14:52,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/7c7d3c5ad1a542778001efb095ebd861, entries=150, sequenceid=182, filesize=30.4 K 2024-12-08T00:14:52,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/c14f199b18f44cf7ae4cf9a370ebe737 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c14f199b18f44cf7ae4cf9a370ebe737 2024-12-08T00:14:53,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c14f199b18f44cf7ae4cf9a370ebe737, entries=150, sequenceid=182, filesize=11.9 K 2024-12-08T00:14:53,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/5ab07aa4bc994324b256afb95a15765e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5ab07aa4bc994324b256afb95a15765e 2024-12-08T00:14:53,006 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5ab07aa4bc994324b256afb95a15765e, entries=150, sequenceid=182, filesize=11.9 K 2024-12-08T00:14:53,007 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c31d593296c0be255698232087c08858 in 1290ms, sequenceid=182, compaction requested=false 2024-12-08T00:14:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-08T00:14:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-08T00:14:53,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-08T00:14:53,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0550 sec 2024-12-08T00:14:53,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.0590 sec 2024-12-08T00:14:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T00:14:53,101 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:53,101 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-08T00:14:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T00:14:53,103 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:53,103 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:53,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T00:14:53,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:53,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733616892594, max timestamp=1733616893232Num uniques 859; 2024-12-08T00:14:53,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:14:53,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733616892594, max timestamp=1733616893232Num uniques 859; 2024-12-08T00:14:53,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:14:53,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733616892594, max timestamp=1733616893232Num uniques 859; 2024-12-08T00:14:53,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:14:53,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=33.54 KB heapSize=81.47 KB 2024-12-08T00:14:53,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:53,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:53,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:53,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:53,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:53,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087b934aaa01c0428dafa1327c896f8a6e_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616893234/Put/seqid=0 2024-12-08T00:14:53,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741992_1168 (size=14794) 2024-12-08T00:14:53,253 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:53,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:53,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:53,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,259 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087b934aaa01c0428dafa1327c896f8a6e_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087b934aaa01c0428dafa1327c896f8a6e_c31d593296c0be255698232087c08858 2024-12-08T00:14:53,261 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/6065e4e9968d46af8157b65c41438343, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:53,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/6065e4e9968d46af8157b65c41438343 is 175, key is test_row_0/A:col10/1733616893234/Put/seqid=0 2024-12-08T00:14:53,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741993_1169 (size=39749) 2024-12-08T00:14:53,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616893262, max timestamp=1733616893266Num uniques 859; 2024-12-08T00:14:53,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616893262, max timestamp=1733616893266Num uniques 859; 2024-12-08T00:14:53,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:53,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:53,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616893262, max timestamp=1733616893266Num uniques 859; 2024-12-08T00:14:53,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:14:53,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616953293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=568, cellSize=26008, totalHeapSize=59312, min timestamp=1733616893274, max timestamp=1733616893297Num uniques 1018; 2024-12-08T00:14:53,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=26008, suffix heap size=133840,new segment heap size=59312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=568 2024-12-08T00:14:53,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733616893284, max timestamp=1733616893297Num uniques 1018; 2024-12-08T00:14:53,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-08T00:14:53,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=568, cellSize=26008, totalHeapSize=59312, min timestamp=1733616893274, max timestamp=1733616893297Num uniques 1018; 2024-12-08T00:14:53,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=26008, suffix heap size=133840,new segment heap size=59312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=568 2024-12-08T00:14:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616953323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616953323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616953324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616953324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616953410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T00:14:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616953431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616953431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616953431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616953431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:53,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:53,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616953621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616953641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616953641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616953641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616953641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,667 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/6065e4e9968d46af8157b65c41438343 2024-12-08T00:14:53,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4ddf5d669d504f15adc9ca8dc71479cb is 50, key is test_row_0/B:col10/1733616893234/Put/seqid=0 2024-12-08T00:14:53,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741994_1170 (size=12151) 2024-12-08T00:14:53,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T00:14:53,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:53,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:53,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:53,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:53,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616953931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616953951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616953951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616953951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:53,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616953951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:54,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:54,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:54,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4ddf5d669d504f15adc9ca8dc71479cb 2024-12-08T00:14:54,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/1950d5a5991d4f8eb1946edb53f078f9 is 50, key is test_row_0/C:col10/1733616893234/Put/seqid=0 2024-12-08T00:14:54,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741995_1171 (size=12151) 2024-12-08T00:14:54,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:54,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:54,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T00:14:54,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616954440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:54,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616954461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:54,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616954461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616954461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616954461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:54,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:54,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:54,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:54,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:54,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/1950d5a5991d4f8eb1946edb53f078f9 2024-12-08T00:14:54,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/6065e4e9968d46af8157b65c41438343 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/6065e4e9968d46af8157b65c41438343 2024-12-08T00:14:54,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/6065e4e9968d46af8157b65c41438343, entries=200, sequenceid=208, filesize=38.8 K 2024-12-08T00:14:54,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4ddf5d669d504f15adc9ca8dc71479cb as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4ddf5d669d504f15adc9ca8dc71479cb 2024-12-08T00:14:54,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4ddf5d669d504f15adc9ca8dc71479cb, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T00:14:54,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/1950d5a5991d4f8eb1946edb53f078f9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/1950d5a5991d4f8eb1946edb53f078f9 2024-12-08T00:14:54,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/1950d5a5991d4f8eb1946edb53f078f9, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T00:14:54,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~121.27 KB/124176, currentSize=149.83 KB/153430 for c31d593296c0be255698232087c08858 in 1274ms, sequenceid=208, compaction requested=true 2024-12-08T00:14:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:54,511 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:54,511 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:54,514 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:54,514 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:54,514 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/B is initiating minor compaction (all files) 2024-12-08T00:14:54,514 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/A is initiating minor compaction (all files) 2024-12-08T00:14:54,514 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/B in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,514 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/A in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,514 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b5017e6bc19c48bd95eaddc9a35a17f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c14f199b18f44cf7ae4cf9a370ebe737, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4ddf5d669d504f15adc9ca8dc71479cb] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=36.0 K 2024-12-08T00:14:54,514 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1fc3a862c8a644a2ae7ad33ea336c85e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/7c7d3c5ad1a542778001efb095ebd861, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/6065e4e9968d46af8157b65c41438343] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=100.0 K 2024-12-08T00:14:54,514 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,514 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1fc3a862c8a644a2ae7ad33ea336c85e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/7c7d3c5ad1a542778001efb095ebd861, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/6065e4e9968d46af8157b65c41438343] 2024-12-08T00:14:54,514 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting b5017e6bc19c48bd95eaddc9a35a17f7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733616890412 2024-12-08T00:14:54,515 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fc3a862c8a644a2ae7ad33ea336c85e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733616890412 2024-12-08T00:14:54,515 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c14f199b18f44cf7ae4cf9a370ebe737, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733616890419 2024-12-08T00:14:54,515 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c7d3c5ad1a542778001efb095ebd861, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733616890419 2024-12-08T00:14:54,515 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ddf5d669d504f15adc9ca8dc71479cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616893232 2024-12-08T00:14:54,515 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6065e4e9968d46af8157b65c41438343, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616893232 2024-12-08T00:14:54,536 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:54,537 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/65221a4242c64f23b284088dba41051e is 50, key is test_row_0/B:col10/1733616893234/Put/seqid=0 2024-12-08T00:14:54,537 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:54,540 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208764b9c2d38694a43a6f70e5ded5e453f_c31d593296c0be255698232087c08858 store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:54,541 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208764b9c2d38694a43a6f70e5ded5e453f_c31d593296c0be255698232087c08858, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:54,541 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208764b9c2d38694a43a6f70e5ded5e453f_c31d593296c0be255698232087c08858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:54,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741996_1172 (size=12663) 2024-12-08T00:14:54,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741997_1173 (size=4469) 2024-12-08T00:14:54,555 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#A#compaction#142 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:54,556 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/3ce09c75fdf245d78767cd458d073395 is 175, key is test_row_0/A:col10/1733616893234/Put/seqid=0 2024-12-08T00:14:54,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741998_1174 (size=31617) 2024-12-08T00:14:54,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T00:14:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,633 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=149.83 KB heapSize=400.94 KB 2024-12-08T00:14:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:54,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120828faf857090f4b078f76e64d126c64cf_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616893322/Put/seqid=0 2024-12-08T00:14:54,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741999_1175 (size=12354) 2024-12-08T00:14:54,952 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/65221a4242c64f23b284088dba41051e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65221a4242c64f23b284088dba41051e 2024-12-08T00:14:54,957 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/B of c31d593296c0be255698232087c08858 into 65221a4242c64f23b284088dba41051e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:54,957 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:54,957 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/B, priority=13, startTime=1733616894511; duration=0sec 2024-12-08T00:14:54,957 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:54,957 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:B 2024-12-08T00:14:54,957 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:54,959 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:54,959 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/C is initiating minor compaction (all files) 2024-12-08T00:14:54,959 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/C in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:54,959 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/e68abefea9ce47f8853498fd5ec75d60, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5ab07aa4bc994324b256afb95a15765e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/1950d5a5991d4f8eb1946edb53f078f9] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=36.0 K 2024-12-08T00:14:54,960 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e68abefea9ce47f8853498fd5ec75d60, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733616890412 2024-12-08T00:14:54,960 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab07aa4bc994324b256afb95a15765e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733616890419 2024-12-08T00:14:54,961 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 1950d5a5991d4f8eb1946edb53f078f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616893232 2024-12-08T00:14:54,968 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:54,969 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/30ffc0521f2b4d9780c9eda0090d57b0 is 50, key is test_row_0/C:col10/1733616893234/Put/seqid=0 2024-12-08T00:14:54,971 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/3ce09c75fdf245d78767cd458d073395 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/3ce09c75fdf245d78767cd458d073395 2024-12-08T00:14:54,976 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/A of c31d593296c0be255698232087c08858 into 3ce09c75fdf245d78767cd458d073395(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:54,976 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:54,976 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/A, priority=13, startTime=1733616894511; duration=0sec 2024-12-08T00:14:54,977 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:54,977 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:A 2024-12-08T00:14:54,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742000_1176 (size=12663) 2024-12-08T00:14:55,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:55,049 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120828faf857090f4b078f76e64d126c64cf_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120828faf857090f4b078f76e64d126c64cf_c31d593296c0be255698232087c08858 2024-12-08T00:14:55,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/adc405c87dc8416a9a2c0909b61989f1, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:55,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/adc405c87dc8416a9a2c0909b61989f1 is 175, key is test_row_0/A:col10/1733616893322/Put/seqid=0 2024-12-08T00:14:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742001_1177 (size=31155) 2024-12-08T00:14:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T00:14:55,391 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/30ffc0521f2b4d9780c9eda0090d57b0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/30ffc0521f2b4d9780c9eda0090d57b0 2024-12-08T00:14:55,395 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/C of c31d593296c0be255698232087c08858 into 30ffc0521f2b4d9780c9eda0090d57b0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:55,395 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:55,395 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/C, priority=13, startTime=1733616894511; duration=0sec 2024-12-08T00:14:55,395 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:55,395 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:C 2024-12-08T00:14:55,455 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/adc405c87dc8416a9a2c0909b61989f1 2024-12-08T00:14:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:55,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4e9715e83d194a7c81f7aa4ad21985a3 is 50, key is test_row_0/B:col10/1733616893322/Put/seqid=0 2024-12-08T00:14:55,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742002_1178 (size=12201) 2024-12-08T00:14:55,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616955473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616955475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,477 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in c31d593296c0be255698232087c08858 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616955476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616955476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616955477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616955580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616955591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616955591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616955591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616955790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616955801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616955801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616955801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:55,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4e9715e83d194a7c81f7aa4ad21985a3 2024-12-08T00:14:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/941df5dd9d364a609b4e506cf775cbb5 is 50, key is test_row_0/C:col10/1733616893322/Put/seqid=0 2024-12-08T00:14:55,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742003_1179 (size=12201) 2024-12-08T00:14:56,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616956101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616956110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616956110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616956111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,284 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/941df5dd9d364a609b4e506cf775cbb5 2024-12-08T00:14:56,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/adc405c87dc8416a9a2c0909b61989f1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/adc405c87dc8416a9a2c0909b61989f1 2024-12-08T00:14:56,293 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/adc405c87dc8416a9a2c0909b61989f1, entries=150, sequenceid=258, filesize=30.4 K 2024-12-08T00:14:56,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/4e9715e83d194a7c81f7aa4ad21985a3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4e9715e83d194a7c81f7aa4ad21985a3 2024-12-08T00:14:56,297 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4e9715e83d194a7c81f7aa4ad21985a3, entries=150, sequenceid=258, filesize=11.9 K 2024-12-08T00:14:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/941df5dd9d364a609b4e506cf775cbb5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/941df5dd9d364a609b4e506cf775cbb5 2024-12-08T00:14:56,302 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/941df5dd9d364a609b4e506cf775cbb5, entries=150, sequenceid=258, filesize=11.9 K 2024-12-08T00:14:56,303 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~149.83 KB/153430, heapSize ~400.89 KB/410512, currentSize=46.96 KB/48090 for c31d593296c0be255698232087c08858 in 1670ms, sequenceid=258, compaction requested=false 2024-12-08T00:14:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-08T00:14:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-08T00:14:56,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-08T00:14:56,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2010 sec 2024-12-08T00:14:56,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.2050 sec 2024-12-08T00:14:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:56,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T00:14:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:56,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ee1aae7b4ee542e7aa7c89692b750cef_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616895475/Put/seqid=0 2024-12-08T00:14:56,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742004_1180 (size=14994) 2024-12-08T00:14:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616956641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616956643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733616896628, max timestamp=1733616896644Num uniques 859; 2024-12-08T00:14:56,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-08T00:14:56,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733616896628, max timestamp=1733616896644Num uniques 859; 2024-12-08T00:14:56,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-08T00:14:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733616896628, max timestamp=1733616896644Num uniques 859; 2024-12-08T00:14:56,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-08T00:14:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616956672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616956673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616956751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616956751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616956781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616956781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616956961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616956961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616956991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616956991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:57,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ee1aae7b4ee542e7aa7c89692b750cef_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ee1aae7b4ee542e7aa7c89692b750cef_c31d593296c0be255698232087c08858 2024-12-08T00:14:57,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/fab51870daaa451ab74f0e97011c4d36, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:57,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/fab51870daaa451ab74f0e97011c4d36 is 175, key is test_row_0/A:col10/1733616895475/Put/seqid=0 2024-12-08T00:14:57,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742005_1181 (size=39949) 2024-12-08T00:14:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T00:14:57,271 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:14:57,271 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:14:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-08T00:14:57,273 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:14:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T00:14:57,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:57,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:14:57,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:14:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616957271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:57,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616957271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616957301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616957301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T00:14:57,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T00:14:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,443 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/fab51870daaa451ab74f0e97011c4d36 2024-12-08T00:14:57,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/9c3f8bdec06849b0bb67c300347571bb is 50, key is test_row_0/B:col10/1733616895475/Put/seqid=0 2024-12-08T00:14:57,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742006_1182 (size=12301) 2024-12-08T00:14:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616957491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:57,492 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in c31d593296c0be255698232087c08858 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T00:14:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T00:14:57,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T00:14:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616957780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:57,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616957780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616957811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616957811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:57,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/9c3f8bdec06849b0bb67c300347571bb 2024-12-08T00:14:57,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/8b79ed5975a246fda8e81af6af7bb997 is 50, key is test_row_0/C:col10/1733616895475/Put/seqid=0 2024-12-08T00:14:57,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742007_1183 (size=12301) 2024-12-08T00:14:57,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/8b79ed5975a246fda8e81af6af7bb997 2024-12-08T00:14:57,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/fab51870daaa451ab74f0e97011c4d36 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/fab51870daaa451ab74f0e97011c4d36 2024-12-08T00:14:57,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/fab51870daaa451ab74f0e97011c4d36, entries=200, sequenceid=272, filesize=39.0 K 2024-12-08T00:14:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/9c3f8bdec06849b0bb67c300347571bb as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/9c3f8bdec06849b0bb67c300347571bb 2024-12-08T00:14:57,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T00:14:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:14:57,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/9c3f8bdec06849b0bb67c300347571bb, entries=150, sequenceid=272, filesize=12.0 K 2024-12-08T00:14:57,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/8b79ed5975a246fda8e81af6af7bb997 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8b79ed5975a246fda8e81af6af7bb997 2024-12-08T00:14:57,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8b79ed5975a246fda8e81af6af7bb997, entries=150, sequenceid=272, filesize=12.0 K 2024-12-08T00:14:57,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for c31d593296c0be255698232087c08858 in 1273ms, sequenceid=272, compaction requested=true 2024-12-08T00:14:57,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:57,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:14:57,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:57,893 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:57,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:14:57,893 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:57,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:57,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:14:57,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:57,894 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:57,894 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:57,894 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/B is initiating minor compaction (all files) 2024-12-08T00:14:57,894 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/B in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,894 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65221a4242c64f23b284088dba41051e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4e9715e83d194a7c81f7aa4ad21985a3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/9c3f8bdec06849b0bb67c300347571bb] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=36.3 K 2024-12-08T00:14:57,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 65221a4242c64f23b284088dba41051e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616893232 2024-12-08T00:14:57,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/A is initiating minor compaction (all files) 2024-12-08T00:14:57,895 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/A in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,895 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/3ce09c75fdf245d78767cd458d073395, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/adc405c87dc8416a9a2c0909b61989f1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/fab51870daaa451ab74f0e97011c4d36] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=100.3 K 2024-12-08T00:14:57,895 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/3ce09c75fdf245d78767cd458d073395, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/adc405c87dc8416a9a2c0909b61989f1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/fab51870daaa451ab74f0e97011c4d36] 2024-12-08T00:14:57,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e9715e83d194a7c81f7aa4ad21985a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733616893321 2024-12-08T00:14:57,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ce09c75fdf245d78767cd458d073395, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616893232 2024-12-08T00:14:57,896 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c3f8bdec06849b0bb67c300347571bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616895475 2024-12-08T00:14:57,896 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting adc405c87dc8416a9a2c0909b61989f1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733616893321 2024-12-08T00:14:57,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting fab51870daaa451ab74f0e97011c4d36, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616895474 2024-12-08T00:14:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T00:14:57,903 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:57,904 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:57,905 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208f6038570d8824beb857e41fb09b6824b_c31d593296c0be255698232087c08858 store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:57,905 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/66900bc518874aa489f34e67d07c4e2b is 50, key is test_row_0/B:col10/1733616895475/Put/seqid=0 2024-12-08T00:14:57,906 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208f6038570d8824beb857e41fb09b6824b_c31d593296c0be255698232087c08858, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:57,907 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f6038570d8824beb857e41fb09b6824b_c31d593296c0be255698232087c08858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:57,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742008_1184 (size=12915) 2024-12-08T00:14:57,915 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/66900bc518874aa489f34e67d07c4e2b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66900bc518874aa489f34e67d07c4e2b 2024-12-08T00:14:57,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742009_1185 (size=4469) 2024-12-08T00:14:57,920 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/B of c31d593296c0be255698232087c08858 into 66900bc518874aa489f34e67d07c4e2b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:57,920 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:57,920 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/B, priority=13, startTime=1733616897893; duration=0sec 2024-12-08T00:14:57,920 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:14:57,920 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:B 2024-12-08T00:14:57,921 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:14:57,922 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:14:57,922 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/C is initiating minor compaction (all files) 2024-12-08T00:14:57,922 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/C in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:57,922 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/30ffc0521f2b4d9780c9eda0090d57b0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/941df5dd9d364a609b4e506cf775cbb5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8b79ed5975a246fda8e81af6af7bb997] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=36.3 K 2024-12-08T00:14:57,923 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 30ffc0521f2b4d9780c9eda0090d57b0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616893232 2024-12-08T00:14:57,923 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#A#compaction#151 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:57,923 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 941df5dd9d364a609b4e506cf775cbb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733616893321 2024-12-08T00:14:57,923 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/88e9c8061a264032b3176492614b0f93 is 175, key is test_row_0/A:col10/1733616895475/Put/seqid=0 2024-12-08T00:14:57,924 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b79ed5975a246fda8e81af6af7bb997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616895475 2024-12-08T00:14:57,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742010_1186 (size=31869) 2024-12-08T00:14:57,934 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#C#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:14:57,934 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/95d7327a386e4f19bbfe0d8a0e03c0ba is 50, key is test_row_0/C:col10/1733616895475/Put/seqid=0 2024-12-08T00:14:57,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742011_1187 (size=12915) 2024-12-08T00:14:58,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T00:14:58,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:58,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-12-08T00:14:58,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:58,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:58,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:58,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:58,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:58,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:14:58,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a54ca42d5f4e4ec4ade5ae99fc219720_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616896670/Put/seqid=0 2024-12-08T00:14:58,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742012_1188 (size=12454) 2024-12-08T00:14:58,335 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/88e9c8061a264032b3176492614b0f93 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/88e9c8061a264032b3176492614b0f93 2024-12-08T00:14:58,340 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/A of c31d593296c0be255698232087c08858 into 88e9c8061a264032b3176492614b0f93(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:58,340 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:58,340 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/A, priority=13, startTime=1733616897893; duration=0sec 2024-12-08T00:14:58,340 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:58,340 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:A 2024-12-08T00:14:58,342 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/95d7327a386e4f19bbfe0d8a0e03c0ba as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/95d7327a386e4f19bbfe0d8a0e03c0ba 2024-12-08T00:14:58,348 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/C of c31d593296c0be255698232087c08858 into 95d7327a386e4f19bbfe0d8a0e03c0ba(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:14:58,348 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:58,348 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/C, priority=13, startTime=1733616897893; duration=0sec 2024-12-08T00:14:58,348 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:14:58,348 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:C 2024-12-08T00:14:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T00:14:58,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:14:58,459 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a54ca42d5f4e4ec4ade5ae99fc219720_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a54ca42d5f4e4ec4ade5ae99fc219720_c31d593296c0be255698232087c08858 2024-12-08T00:14:58,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/8b2eae076c2b42d7abe44183af7a23d7, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:14:58,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/8b2eae076c2b42d7abe44183af7a23d7 is 175, key is test_row_0/A:col10/1733616896670/Put/seqid=0 2024-12-08T00:14:58,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742013_1189 (size=31255) 2024-12-08T00:14:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:58,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. as already flushing 2024-12-08T00:14:58,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616958814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:58,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:58,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616958814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:58,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616958831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:58,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616958831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:58,834 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in c31d593296c0be255698232087c08858 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:58,835 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in c31d593296c0be255698232087c08858 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:14:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:58,864 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/8b2eae076c2b42d7abe44183af7a23d7 2024-12-08T00:14:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/e808e0127445496eae3e9785f8ab33fc is 50, key is test_row_0/B:col10/1733616896670/Put/seqid=0 2024-12-08T00:14:58,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742014_1190 (size=12301) 2024-12-08T00:14:58,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616958921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:58,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616958921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:59,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:59,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616959130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616959131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:59,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:59,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:59,275 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/e808e0127445496eae3e9785f8ab33fc 2024-12-08T00:14:59,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/792194bc5a6e42609f2cd6e64158dd28 is 50, key is test_row_0/C:col10/1733616896670/Put/seqid=0 2024-12-08T00:14:59,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742015_1191 (size=12301) 2024-12-08T00:14:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T00:14:59,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616959441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:59,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616959441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:59,689 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/792194bc5a6e42609f2cd6e64158dd28 2024-12-08T00:14:59,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/8b2eae076c2b42d7abe44183af7a23d7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/8b2eae076c2b42d7abe44183af7a23d7 2024-12-08T00:14:59,698 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/8b2eae076c2b42d7abe44183af7a23d7, entries=150, sequenceid=306, filesize=30.5 K 2024-12-08T00:14:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/e808e0127445496eae3e9785f8ab33fc as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/e808e0127445496eae3e9785f8ab33fc 2024-12-08T00:14:59,703 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/e808e0127445496eae3e9785f8ab33fc, entries=150, sequenceid=306, filesize=12.0 K 2024-12-08T00:14:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/792194bc5a6e42609f2cd6e64158dd28 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/792194bc5a6e42609f2cd6e64158dd28 2024-12-08T00:14:59,708 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/792194bc5a6e42609f2cd6e64158dd28, entries=150, sequenceid=306, filesize=12.0 K 2024-12-08T00:14:59,709 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=60.38 KB/61830 for c31d593296c0be255698232087c08858 in 1670ms, sequenceid=306, compaction requested=false 2024-12-08T00:14:59,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:14:59,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:14:59,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-08T00:14:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-08T00:14:59,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-08T00:14:59,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4360 sec 2024-12-08T00:14:59,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.4410 sec 2024-12-08T00:14:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:14:59,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:14:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:14:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:14:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:14:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:14:59,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208aadda5d5aad64ad3a189175a79a923ff_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616899952/Put/seqid=0 2024-12-08T00:14:59,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742016_1192 (size=17534) 2024-12-08T00:14:59,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616959987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:14:59,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:14:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616959988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:14:59,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:14:59,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:00,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616960091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:00,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616960101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:00,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616960301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:00,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616960311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:00,327 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-12-08T00:15:00,327 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:00,327 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:00,327 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:00,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:00,328 INFO [Thread-585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-585. 2024-12-08T00:15:00,328 DEBUG [Thread-585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:00,328 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:00,328 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:00,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:00,332 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-12-08T00:15:00,332 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:00,332 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:00,332 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:00,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:00,335 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-12-08T00:15:00,335 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:00,335 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:00,335 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:00,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:00,375 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:00,382 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208aadda5d5aad64ad3a189175a79a923ff_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208aadda5d5aad64ad3a189175a79a923ff_c31d593296c0be255698232087c08858 2024-12-08T00:15:00,384 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/5f433deb165b4a5485345c6b28418a9e, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:15:00,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/5f433deb165b4a5485345c6b28418a9e is 175, key is test_row_0/A:col10/1733616899952/Put/seqid=0 2024-12-08T00:15:00,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742017_1193 (size=48639) 2024-12-08T00:15:00,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616960612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:00,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616960621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:00,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/5f433deb165b4a5485345c6b28418a9e 2024-12-08T00:15:00,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/56dec4ea68f04eccb95dbea30027b6b4 is 50, key is test_row_0/B:col10/1733616899952/Put/seqid=0 2024-12-08T00:15:00,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742018_1194 (size=12301) 2024-12-08T00:15:00,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:00,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48896 deadline: 1733616960861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48930 deadline: 1733616960861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:00,863 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in c31d593296c0be255698232087c08858 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:00,863 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in c31d593296c0be255698232087c08858 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:01,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48916 deadline: 1733616961120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48928 deadline: 1733616961131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:01,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/56dec4ea68f04eccb95dbea30027b6b4 2024-12-08T00:15:01,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/9ba7237412704dd996a69b67b5fdafbe is 50, key is test_row_0/C:col10/1733616899952/Put/seqid=0 2024-12-08T00:15:01,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742019_1195 (size=12301) 2024-12-08T00:15:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T00:15:01,442 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:01,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48900 deadline: 1733616961512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:01,514 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in c31d593296c0be255698232087c08858 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:01,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/9ba7237412704dd996a69b67b5fdafbe 2024-12-08T00:15:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/5f433deb165b4a5485345c6b28418a9e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/5f433deb165b4a5485345c6b28418a9e 2024-12-08T00:15:01,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/5f433deb165b4a5485345c6b28418a9e, entries=250, sequenceid=322, filesize=47.5 K 2024-12-08T00:15:01,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/56dec4ea68f04eccb95dbea30027b6b4 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/56dec4ea68f04eccb95dbea30027b6b4 2024-12-08T00:15:01,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/56dec4ea68f04eccb95dbea30027b6b4, entries=150, sequenceid=322, filesize=12.0 K 2024-12-08T00:15:01,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/9ba7237412704dd996a69b67b5fdafbe as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/9ba7237412704dd996a69b67b5fdafbe 2024-12-08T00:15:01,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/9ba7237412704dd996a69b67b5fdafbe, entries=150, sequenceid=322, filesize=12.0 K 2024-12-08T00:15:01,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c31d593296c0be255698232087c08858 in 1697ms, sequenceid=322, compaction requested=true 2024-12-08T00:15:01,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:15:01,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:01,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:01,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:01,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:01,650 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:01,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c31d593296c0be255698232087c08858:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:01,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:01,650 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:01,651 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:01,651 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111763 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:01,651 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/A is initiating minor compaction (all files) 2024-12-08T00:15:01,651 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/B is initiating minor compaction (all files) 2024-12-08T00:15:01,651 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/A in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:15:01,651 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/B in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:15:01,651 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/88e9c8061a264032b3176492614b0f93, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/8b2eae076c2b42d7abe44183af7a23d7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/5f433deb165b4a5485345c6b28418a9e] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=109.1 K 2024-12-08T00:15:01,651 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66900bc518874aa489f34e67d07c4e2b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/e808e0127445496eae3e9785f8ab33fc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/56dec4ea68f04eccb95dbea30027b6b4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=36.6 K 2024-12-08T00:15:01,651 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:15:01,651 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/88e9c8061a264032b3176492614b0f93, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/8b2eae076c2b42d7abe44183af7a23d7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/5f433deb165b4a5485345c6b28418a9e] 2024-12-08T00:15:01,651 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 66900bc518874aa489f34e67d07c4e2b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616895475 2024-12-08T00:15:01,652 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88e9c8061a264032b3176492614b0f93, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616895475 2024-12-08T00:15:01,652 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e808e0127445496eae3e9785f8ab33fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733616896666 2024-12-08T00:15:01,652 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b2eae076c2b42d7abe44183af7a23d7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733616896666 2024-12-08T00:15:01,652 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 56dec4ea68f04eccb95dbea30027b6b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733616898807 2024-12-08T00:15:01,652 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f433deb165b4a5485345c6b28418a9e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733616898806 2024-12-08T00:15:01,661 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:01,662 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:15:01,662 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/8596e9e9b1614376ae782d53669c1caa is 50, key is test_row_0/B:col10/1733616899952/Put/seqid=0 2024-12-08T00:15:01,664 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412088a667028330545c1a2845db171b6a4a3_c31d593296c0be255698232087c08858 store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:15:01,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742020_1196 (size=13017) 2024-12-08T00:15:01,682 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412088a667028330545c1a2845db171b6a4a3_c31d593296c0be255698232087c08858, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:15:01,682 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088a667028330545c1a2845db171b6a4a3_c31d593296c0be255698232087c08858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:15:01,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742021_1197 (size=4469) 2024-12-08T00:15:02,089 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#A#compaction#160 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:02,090 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/c2b9ed0298c249bb917e3e3f84b57005 is 175, key is test_row_0/A:col10/1733616899952/Put/seqid=0 2024-12-08T00:15:02,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742022_1198 (size=31971) 2024-12-08T00:15:02,127 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/8596e9e9b1614376ae782d53669c1caa as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/8596e9e9b1614376ae782d53669c1caa 2024-12-08T00:15:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c31d593296c0be255698232087c08858 2024-12-08T00:15:02,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T00:15:02,134 INFO [Thread-578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-578. 2024-12-08T00:15:02,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:15:02,134 DEBUG [Thread-578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:02,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:02,134 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:02,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:15:02,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:02,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:15:02,134 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:02,134 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:02,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:02,137 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/B of c31d593296c0be255698232087c08858 into 8596e9e9b1614376ae782d53669c1caa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:02,137 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:15:02,137 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/B, priority=13, startTime=1733616901650; duration=0sec 2024-12-08T00:15:02,138 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:02,138 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:B 2024-12-08T00:15:02,138 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:02,139 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:02,139 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c31d593296c0be255698232087c08858/C is initiating minor compaction (all files) 2024-12-08T00:15:02,139 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c31d593296c0be255698232087c08858/C in TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:15:02,140 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/95d7327a386e4f19bbfe0d8a0e03c0ba, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/792194bc5a6e42609f2cd6e64158dd28, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/9ba7237412704dd996a69b67b5fdafbe] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp, totalSize=36.6 K 2024-12-08T00:15:02,140 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d7327a386e4f19bbfe0d8a0e03c0ba, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616895475 2024-12-08T00:15:02,141 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 792194bc5a6e42609f2cd6e64158dd28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733616896666 2024-12-08T00:15:02,141 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba7237412704dd996a69b67b5fdafbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733616898807 2024-12-08T00:15:02,143 INFO [Thread-582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-582. 2024-12-08T00:15:02,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c92f720e196749f69a6703ba16e62f21_c31d593296c0be255698232087c08858 is 50, key is test_row_0/A:col10/1733616899975/Put/seqid=0 2024-12-08T00:15:02,143 DEBUG [Thread-582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:02,143 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:02,143 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:02,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:02,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742023_1199 (size=12454) 2024-12-08T00:15:02,150 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c31d593296c0be255698232087c08858#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:02,150 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/dd4cea92b6d14b2da2c152572060ec33 is 50, key is test_row_0/C:col10/1733616899952/Put/seqid=0 2024-12-08T00:15:02,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742024_1200 (size=13017) 2024-12-08T00:15:02,538 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/c2b9ed0298c249bb917e3e3f84b57005 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/c2b9ed0298c249bb917e3e3f84b57005 2024-12-08T00:15:02,544 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/A of c31d593296c0be255698232087c08858 into c2b9ed0298c249bb917e3e3f84b57005(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:02,544 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:15:02,544 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/A, priority=13, startTime=1733616901650; duration=0sec 2024-12-08T00:15:02,544 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:02,544 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:A 2024-12-08T00:15:02,547 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:02,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c92f720e196749f69a6703ba16e62f21_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c92f720e196749f69a6703ba16e62f21_c31d593296c0be255698232087c08858 2024-12-08T00:15:02,553 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/df0eec9f0780464ca60c57313562aff9, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:15:02,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/df0eec9f0780464ca60c57313562aff9 is 175, key is test_row_0/A:col10/1733616899975/Put/seqid=0 2024-12-08T00:15:02,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742025_1201 (size=31255) 2024-12-08T00:15:02,559 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/dd4cea92b6d14b2da2c152572060ec33 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/dd4cea92b6d14b2da2c152572060ec33 2024-12-08T00:15:02,564 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c31d593296c0be255698232087c08858/C of c31d593296c0be255698232087c08858 into dd4cea92b6d14b2da2c152572060ec33(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:02,564 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:15:02,564 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., storeName=c31d593296c0be255698232087c08858/C, priority=13, startTime=1733616901650; duration=0sec 2024-12-08T00:15:02,564 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:02,564 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c31d593296c0be255698232087c08858:C 2024-12-08T00:15:02,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/df0eec9f0780464ca60c57313562aff9 2024-12-08T00:15:02,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/1e753cf4d164431aa249772d62333a05 is 50, key is test_row_0/B:col10/1733616899975/Put/seqid=0 2024-12-08T00:15:02,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742026_1202 (size=12301) 2024-12-08T00:15:03,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/1e753cf4d164431aa249772d62333a05 2024-12-08T00:15:03,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/46cf0e9e9e31426d9043de6fb01cde2a is 50, key is test_row_0/C:col10/1733616899975/Put/seqid=0 2024-12-08T00:15:03,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742027_1203 (size=12301) 2024-12-08T00:15:03,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/46cf0e9e9e31426d9043de6fb01cde2a 2024-12-08T00:15:03,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/df0eec9f0780464ca60c57313562aff9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/df0eec9f0780464ca60c57313562aff9 2024-12-08T00:15:03,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/df0eec9f0780464ca60c57313562aff9, entries=150, sequenceid=341, filesize=30.5 K 2024-12-08T00:15:03,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/1e753cf4d164431aa249772d62333a05 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/1e753cf4d164431aa249772d62333a05 2024-12-08T00:15:03,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/1e753cf4d164431aa249772d62333a05, entries=150, sequenceid=341, filesize=12.0 K 2024-12-08T00:15:03,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/46cf0e9e9e31426d9043de6fb01cde2a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/46cf0e9e9e31426d9043de6fb01cde2a 2024-12-08T00:15:03,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/46cf0e9e9e31426d9043de6fb01cde2a, entries=150, sequenceid=341, filesize=12.0 K 2024-12-08T00:15:03,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for c31d593296c0be255698232087c08858 in 1695ms, sequenceid=341, compaction requested=false 2024-12-08T00:15:03,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c31d593296c0be255698232087c08858: 2024-12-08T00:15:04,886 INFO [Thread-576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-576. 2024-12-08T00:15:04,886 INFO [Thread-574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-574. 2024-12-08T00:15:04,886 DEBUG [Thread-576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:04,886 DEBUG [Thread-574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:04,886 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:04,886 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:04,887 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:04,887 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:04,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:04,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:06,243 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T00:15:11,542 INFO [Thread-580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-580. 2024-12-08T00:15:11,542 DEBUG [Thread-580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:11,542 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:11,542 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:11,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4565 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4367 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1917 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5748 rows 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1899 2024-12-08T00:15:11,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5692 rows 2024-12-08T00:15:11,543 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T00:15:11,543 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T00:15:11,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:11,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:11,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:11,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T00:15:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T00:15:11,549 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616911548"}]},"ts":"1733616911548"} 2024-12-08T00:15:11,551 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T00:15:11,551 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T00:15:11,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T00:15:11,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, UNASSIGN}] 2024-12-08T00:15:11,555 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, UNASSIGN 2024-12-08T00:15:11,556 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=c31d593296c0be255698232087c08858, regionState=CLOSING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:11,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, UNASSIGN because future has completed 2024-12-08T00:15:11,558 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T00:15:11,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:15:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T00:15:11,711 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close c31d593296c0be255698232087c08858 2024-12-08T00:15:11,711 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T00:15:11,711 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing c31d593296c0be255698232087c08858, disabling compactions & flushes 2024-12-08T00:15:11,711 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:15:11,711 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:15:11,711 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. after waiting 0 ms 2024-12-08T00:15:11,711 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:15:11,711 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing c31d593296c0be255698232087c08858 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-08T00:15:11,712 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=A 2024-12-08T00:15:11,712 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:11,712 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=B 2024-12-08T00:15:11,712 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:11,712 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c31d593296c0be255698232087c08858, store=C 2024-12-08T00:15:11,712 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:11,718 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e663933101b34de295aec244a07d75e9_c31d593296c0be255698232087c08858 is 50, key is test_row_1/A:col10/1733616911541/Put/seqid=0 2024-12-08T00:15:11,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742028_1204 (size=9914) 2024-12-08T00:15:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T00:15:12,129 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:12,134 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e663933101b34de295aec244a07d75e9_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e663933101b34de295aec244a07d75e9_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,135 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/611e57481ab1459082f39b03edb0803b, store: [table=TestAcidGuarantees family=A region=c31d593296c0be255698232087c08858] 2024-12-08T00:15:12,136 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/611e57481ab1459082f39b03edb0803b is 175, key is test_row_1/A:col10/1733616911541/Put/seqid=0 2024-12-08T00:15:12,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742029_1205 (size=22561) 2024-12-08T00:15:12,144 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/611e57481ab1459082f39b03edb0803b 2024-12-08T00:15:12,163 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/c72d2bef62fb44c2ae7f1c5077f87de0 is 50, key is test_row_1/B:col10/1733616911541/Put/seqid=0 2024-12-08T00:15:12,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742030_1206 (size=9857) 2024-12-08T00:15:12,169 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/c72d2bef62fb44c2ae7f1c5077f87de0 2024-12-08T00:15:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T00:15:12,194 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/f34b3b52d8af420b95f580e32ada5d93 is 50, key is test_row_1/C:col10/1733616911541/Put/seqid=0 2024-12-08T00:15:12,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742031_1207 (size=9857) 2024-12-08T00:15:12,217 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/f34b3b52d8af420b95f580e32ada5d93 2024-12-08T00:15:12,225 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/A/611e57481ab1459082f39b03edb0803b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/611e57481ab1459082f39b03edb0803b 2024-12-08T00:15:12,234 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/611e57481ab1459082f39b03edb0803b, entries=100, sequenceid=351, filesize=22.0 K 2024-12-08T00:15:12,235 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/B/c72d2bef62fb44c2ae7f1c5077f87de0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c72d2bef62fb44c2ae7f1c5077f87de0 2024-12-08T00:15:12,242 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c72d2bef62fb44c2ae7f1c5077f87de0, entries=100, sequenceid=351, filesize=9.6 K 2024-12-08T00:15:12,243 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/.tmp/C/f34b3b52d8af420b95f580e32ada5d93 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/f34b3b52d8af420b95f580e32ada5d93 2024-12-08T00:15:12,250 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/f34b3b52d8af420b95f580e32ada5d93, entries=100, sequenceid=351, filesize=9.6 K 2024-12-08T00:15:12,251 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c31d593296c0be255698232087c08858 in 540ms, sequenceid=351, compaction requested=true 2024-12-08T00:15:12,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/92c54d1427ef4eeea8d587f3732d30f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/91f4262913f74147a7829566621732e8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/e4c80853a78b4fc28a08e5e7c430b23d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/c5c4d0335f5f46258d5613c55e8a8d7a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/87f811767e434925aa8d703680a8af22, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1f28b4a43ab94fc6b9c44c508cbf4645, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1d95e4cb373f49439deb7f6e8ecbc6ea, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/593c14b56b8e4c2b90b8bc1161c3d482, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/27f3b8e06c004c18b9b09e03114c6b95, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/080717c38dd54ae0af98189c2f1aa55e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/464c14413fec4381aa5bebdedc317d4a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/f5a972700da34f959e724f4bd18de9d2, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1fc3a862c8a644a2ae7ad33ea336c85e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/7c7d3c5ad1a542778001efb095ebd861, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/6065e4e9968d46af8157b65c41438343, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/3ce09c75fdf245d78767cd458d073395, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/adc405c87dc8416a9a2c0909b61989f1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/fab51870daaa451ab74f0e97011c4d36, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/88e9c8061a264032b3176492614b0f93, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/8b2eae076c2b42d7abe44183af7a23d7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/5f433deb165b4a5485345c6b28418a9e] to archive 2024-12-08T00:15:12,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:15:12,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/92c54d1427ef4eeea8d587f3732d30f7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/92c54d1427ef4eeea8d587f3732d30f7 2024-12-08T00:15:12,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/91f4262913f74147a7829566621732e8 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/91f4262913f74147a7829566621732e8 2024-12-08T00:15:12,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/e4c80853a78b4fc28a08e5e7c430b23d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/e4c80853a78b4fc28a08e5e7c430b23d 2024-12-08T00:15:12,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/c5c4d0335f5f46258d5613c55e8a8d7a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/c5c4d0335f5f46258d5613c55e8a8d7a 2024-12-08T00:15:12,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/87f811767e434925aa8d703680a8af22 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/87f811767e434925aa8d703680a8af22 2024-12-08T00:15:12,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1f28b4a43ab94fc6b9c44c508cbf4645 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1f28b4a43ab94fc6b9c44c508cbf4645 2024-12-08T00:15:12,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1d95e4cb373f49439deb7f6e8ecbc6ea to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1d95e4cb373f49439deb7f6e8ecbc6ea 2024-12-08T00:15:12,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/593c14b56b8e4c2b90b8bc1161c3d482 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/593c14b56b8e4c2b90b8bc1161c3d482 2024-12-08T00:15:12,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/27f3b8e06c004c18b9b09e03114c6b95 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/27f3b8e06c004c18b9b09e03114c6b95 2024-12-08T00:15:12,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/080717c38dd54ae0af98189c2f1aa55e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/080717c38dd54ae0af98189c2f1aa55e 2024-12-08T00:15:12,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/464c14413fec4381aa5bebdedc317d4a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/464c14413fec4381aa5bebdedc317d4a 2024-12-08T00:15:12,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/f5a972700da34f959e724f4bd18de9d2 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/f5a972700da34f959e724f4bd18de9d2 2024-12-08T00:15:12,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1fc3a862c8a644a2ae7ad33ea336c85e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/1fc3a862c8a644a2ae7ad33ea336c85e 2024-12-08T00:15:12,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/7c7d3c5ad1a542778001efb095ebd861 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/7c7d3c5ad1a542778001efb095ebd861 2024-12-08T00:15:12,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/6065e4e9968d46af8157b65c41438343 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/6065e4e9968d46af8157b65c41438343 2024-12-08T00:15:12,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/3ce09c75fdf245d78767cd458d073395 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/3ce09c75fdf245d78767cd458d073395 2024-12-08T00:15:12,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/adc405c87dc8416a9a2c0909b61989f1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/adc405c87dc8416a9a2c0909b61989f1 2024-12-08T00:15:12,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/fab51870daaa451ab74f0e97011c4d36 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/fab51870daaa451ab74f0e97011c4d36 2024-12-08T00:15:12,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/88e9c8061a264032b3176492614b0f93 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/88e9c8061a264032b3176492614b0f93 2024-12-08T00:15:12,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/8b2eae076c2b42d7abe44183af7a23d7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/8b2eae076c2b42d7abe44183af7a23d7 2024-12-08T00:15:12,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/5f433deb165b4a5485345c6b28418a9e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/5f433deb165b4a5485345c6b28418a9e 2024-12-08T00:15:12,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/98c8a479affd4ff5ab9801372e6eaa00, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4d97756d30ba41dfa1348db000eebf05, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/cad446863c044f3a98fbf88dadec1094, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0e4c25c5367a44d4b3cd6bf95b3458cc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4fd980b1c05b47a2b4204f3c77287489, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0ae2f2f5792d4910a03b11eee708f02f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66c66656c35c4a63aecac591471c78a0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c124d7d9402747dd96d7e51364627067, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/bfc16206580b4b8d9b9d5ea43e6cf920, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/71c8ce5023464fafb0328cd9db58dae5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65c308356cf540edbfce719d409a9c06, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b5017e6bc19c48bd95eaddc9a35a17f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b0694d4657b0475ebbc1d75880af079c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c14f199b18f44cf7ae4cf9a370ebe737, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65221a4242c64f23b284088dba41051e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4ddf5d669d504f15adc9ca8dc71479cb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4e9715e83d194a7c81f7aa4ad21985a3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66900bc518874aa489f34e67d07c4e2b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/9c3f8bdec06849b0bb67c300347571bb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/e808e0127445496eae3e9785f8ab33fc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/56dec4ea68f04eccb95dbea30027b6b4] to archive 2024-12-08T00:15:12,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:15:12,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/98c8a479affd4ff5ab9801372e6eaa00 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/98c8a479affd4ff5ab9801372e6eaa00 2024-12-08T00:15:12,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4d97756d30ba41dfa1348db000eebf05 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4d97756d30ba41dfa1348db000eebf05 2024-12-08T00:15:12,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/cad446863c044f3a98fbf88dadec1094 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/cad446863c044f3a98fbf88dadec1094 2024-12-08T00:15:12,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0e4c25c5367a44d4b3cd6bf95b3458cc to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0e4c25c5367a44d4b3cd6bf95b3458cc 2024-12-08T00:15:12,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4fd980b1c05b47a2b4204f3c77287489 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4fd980b1c05b47a2b4204f3c77287489 2024-12-08T00:15:12,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0ae2f2f5792d4910a03b11eee708f02f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/0ae2f2f5792d4910a03b11eee708f02f 2024-12-08T00:15:12,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66c66656c35c4a63aecac591471c78a0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66c66656c35c4a63aecac591471c78a0 2024-12-08T00:15:12,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c124d7d9402747dd96d7e51364627067 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c124d7d9402747dd96d7e51364627067 2024-12-08T00:15:12,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/bfc16206580b4b8d9b9d5ea43e6cf920 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/bfc16206580b4b8d9b9d5ea43e6cf920 2024-12-08T00:15:12,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/71c8ce5023464fafb0328cd9db58dae5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/71c8ce5023464fafb0328cd9db58dae5 2024-12-08T00:15:12,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65c308356cf540edbfce719d409a9c06 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65c308356cf540edbfce719d409a9c06 2024-12-08T00:15:12,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b5017e6bc19c48bd95eaddc9a35a17f7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b5017e6bc19c48bd95eaddc9a35a17f7 2024-12-08T00:15:12,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b0694d4657b0475ebbc1d75880af079c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/b0694d4657b0475ebbc1d75880af079c 2024-12-08T00:15:12,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c14f199b18f44cf7ae4cf9a370ebe737 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c14f199b18f44cf7ae4cf9a370ebe737 2024-12-08T00:15:12,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65221a4242c64f23b284088dba41051e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/65221a4242c64f23b284088dba41051e 2024-12-08T00:15:12,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4ddf5d669d504f15adc9ca8dc71479cb to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4ddf5d669d504f15adc9ca8dc71479cb 2024-12-08T00:15:12,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4e9715e83d194a7c81f7aa4ad21985a3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/4e9715e83d194a7c81f7aa4ad21985a3 2024-12-08T00:15:12,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66900bc518874aa489f34e67d07c4e2b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/66900bc518874aa489f34e67d07c4e2b 2024-12-08T00:15:12,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/9c3f8bdec06849b0bb67c300347571bb to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/9c3f8bdec06849b0bb67c300347571bb 2024-12-08T00:15:12,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/e808e0127445496eae3e9785f8ab33fc to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/e808e0127445496eae3e9785f8ab33fc 2024-12-08T00:15:12,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/56dec4ea68f04eccb95dbea30027b6b4 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/56dec4ea68f04eccb95dbea30027b6b4 2024-12-08T00:15:12,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5a32b0a024c04401bdda190765989b00, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/642798b76461400b853fb58fe5a82eb1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8fb958bcc8df4a058189fa730ffc3561, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5923f006a6244407bcd9b7c226eb953a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/43aec6df38864ccf800ebd4e8034423d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/2f19a508cc7a4e3e9382eab086ac5c2f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c4812367ceb34dd2bc3fb1e29ea4bf28, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/d4db4a1367744fc4a2f0a41fcfd1a9b6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c541287fd67e430394fea62a859075a5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/cc2a619cf45a4c07945f9b223cd4238c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/63702917290542aeba0cd5c32ad15030, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/e68abefea9ce47f8853498fd5ec75d60, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/ba757dbb280241118b10e6cc53577ab1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5ab07aa4bc994324b256afb95a15765e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/30ffc0521f2b4d9780c9eda0090d57b0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/1950d5a5991d4f8eb1946edb53f078f9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/941df5dd9d364a609b4e506cf775cbb5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/95d7327a386e4f19bbfe0d8a0e03c0ba, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8b79ed5975a246fda8e81af6af7bb997, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/792194bc5a6e42609f2cd6e64158dd28, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/9ba7237412704dd996a69b67b5fdafbe] to archive 2024-12-08T00:15:12,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:15:12,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5a32b0a024c04401bdda190765989b00 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5a32b0a024c04401bdda190765989b00 2024-12-08T00:15:12,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/642798b76461400b853fb58fe5a82eb1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/642798b76461400b853fb58fe5a82eb1 2024-12-08T00:15:12,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8fb958bcc8df4a058189fa730ffc3561 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8fb958bcc8df4a058189fa730ffc3561 2024-12-08T00:15:12,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5923f006a6244407bcd9b7c226eb953a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5923f006a6244407bcd9b7c226eb953a 2024-12-08T00:15:12,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/43aec6df38864ccf800ebd4e8034423d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/43aec6df38864ccf800ebd4e8034423d 2024-12-08T00:15:12,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/2f19a508cc7a4e3e9382eab086ac5c2f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/2f19a508cc7a4e3e9382eab086ac5c2f 2024-12-08T00:15:12,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c4812367ceb34dd2bc3fb1e29ea4bf28 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c4812367ceb34dd2bc3fb1e29ea4bf28 2024-12-08T00:15:12,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/d4db4a1367744fc4a2f0a41fcfd1a9b6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/d4db4a1367744fc4a2f0a41fcfd1a9b6 2024-12-08T00:15:12,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c541287fd67e430394fea62a859075a5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/c541287fd67e430394fea62a859075a5 2024-12-08T00:15:12,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/cc2a619cf45a4c07945f9b223cd4238c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/cc2a619cf45a4c07945f9b223cd4238c 2024-12-08T00:15:12,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/63702917290542aeba0cd5c32ad15030 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/63702917290542aeba0cd5c32ad15030 2024-12-08T00:15:12,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/e68abefea9ce47f8853498fd5ec75d60 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/e68abefea9ce47f8853498fd5ec75d60 2024-12-08T00:15:12,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/ba757dbb280241118b10e6cc53577ab1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/ba757dbb280241118b10e6cc53577ab1 2024-12-08T00:15:12,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5ab07aa4bc994324b256afb95a15765e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/5ab07aa4bc994324b256afb95a15765e 2024-12-08T00:15:12,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/30ffc0521f2b4d9780c9eda0090d57b0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/30ffc0521f2b4d9780c9eda0090d57b0 2024-12-08T00:15:12,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/1950d5a5991d4f8eb1946edb53f078f9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/1950d5a5991d4f8eb1946edb53f078f9 2024-12-08T00:15:12,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/941df5dd9d364a609b4e506cf775cbb5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/941df5dd9d364a609b4e506cf775cbb5 2024-12-08T00:15:12,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/95d7327a386e4f19bbfe0d8a0e03c0ba to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/95d7327a386e4f19bbfe0d8a0e03c0ba 2024-12-08T00:15:12,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8b79ed5975a246fda8e81af6af7bb997 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/8b79ed5975a246fda8e81af6af7bb997 2024-12-08T00:15:12,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/792194bc5a6e42609f2cd6e64158dd28 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/792194bc5a6e42609f2cd6e64158dd28 2024-12-08T00:15:12,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/9ba7237412704dd996a69b67b5fdafbe to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/9ba7237412704dd996a69b67b5fdafbe 2024-12-08T00:15:12,408 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/recovered.edits/354.seqid, newMaxSeqId=354, maxSeqId=4 2024-12-08T00:15:12,408 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. 2024-12-08T00:15:12,409 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for c31d593296c0be255698232087c08858: Waiting for close lock at 1733616911711Running coprocessor pre-close hooks at 1733616911711Disabling compacts and flushes for region at 1733616911711Disabling writes for close at 1733616911711Obtaining lock to block concurrent updates at 1733616911711Preparing flush snapshotting stores in c31d593296c0be255698232087c08858 at 1733616911711Finished memstore snapshotting TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733616911712 (+1 ms)Flushing stores of TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858. at 1733616911712Flushing c31d593296c0be255698232087c08858/A: creating writer at 1733616911712Flushing mob file c31d593296c0be255698232087c08858/A: appending metadata at 1733616911718 (+6 ms)Flushing mob file c31d593296c0be255698232087c08858/A: closing flushed file at 1733616911718Flushing c31d593296c0be255698232087c08858/A: appending metadata at 1733616912135 (+417 ms)Flushing c31d593296c0be255698232087c08858/A: closing flushed file at 1733616912135Flushing c31d593296c0be255698232087c08858/B: creating writer at 1733616912150 (+15 ms)Flushing c31d593296c0be255698232087c08858/B: appending metadata at 1733616912162 (+12 ms)Flushing c31d593296c0be255698232087c08858/B: closing flushed file at 1733616912162Flushing c31d593296c0be255698232087c08858/C: creating writer at 1733616912176 (+14 ms)Flushing c31d593296c0be255698232087c08858/C: appending metadata at 1733616912194 (+18 ms)Flushing c31d593296c0be255698232087c08858/C: closing flushed file at 1733616912194Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ab62e8b: reopening flushed file at 1733616912223 (+29 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bb706ef: reopening flushed file at 1733616912234 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dd40212: reopening flushed file at 1733616912242 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c31d593296c0be255698232087c08858 in 540ms, sequenceid=351, compaction requested=true at 1733616912251 (+9 ms)Writing region close event to WAL at 1733616912404 (+153 ms)Running coprocessor post-close hooks at 1733616912408 (+4 ms)Closed at 1733616912408 2024-12-08T00:15:12,411 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed c31d593296c0be255698232087c08858 2024-12-08T00:15:12,411 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=c31d593296c0be255698232087c08858, regionState=CLOSED 2024-12-08T00:15:12,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:15:12,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-08T00:15:12,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure c31d593296c0be255698232087c08858, server=c1a8cf8d4cfa,36021,1733616847970 in 855 msec 2024-12-08T00:15:12,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-08T00:15:12,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c31d593296c0be255698232087c08858, UNASSIGN in 862 msec 2024-12-08T00:15:12,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-08T00:15:12,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 867 msec 2024-12-08T00:15:12,422 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616912422"}]},"ts":"1733616912422"} 2024-12-08T00:15:12,423 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T00:15:12,423 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T00:15:12,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 880 msec 2024-12-08T00:15:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T00:15:12,682 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:12,682 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T00:15:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:12,685 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T00:15:12,687 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:12,692 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858 2024-12-08T00:15:12,696 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/recovered.edits] 2024-12-08T00:15:12,699 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/611e57481ab1459082f39b03edb0803b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/611e57481ab1459082f39b03edb0803b 2024-12-08T00:15:12,700 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/c2b9ed0298c249bb917e3e3f84b57005 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/c2b9ed0298c249bb917e3e3f84b57005 2024-12-08T00:15:12,702 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/df0eec9f0780464ca60c57313562aff9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/A/df0eec9f0780464ca60c57313562aff9 2024-12-08T00:15:12,703 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/1e753cf4d164431aa249772d62333a05 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/1e753cf4d164431aa249772d62333a05 2024-12-08T00:15:12,704 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/8596e9e9b1614376ae782d53669c1caa to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/8596e9e9b1614376ae782d53669c1caa 2024-12-08T00:15:12,705 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c72d2bef62fb44c2ae7f1c5077f87de0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/B/c72d2bef62fb44c2ae7f1c5077f87de0 2024-12-08T00:15:12,707 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/46cf0e9e9e31426d9043de6fb01cde2a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/46cf0e9e9e31426d9043de6fb01cde2a 2024-12-08T00:15:12,708 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/dd4cea92b6d14b2da2c152572060ec33 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/dd4cea92b6d14b2da2c152572060ec33 2024-12-08T00:15:12,709 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/f34b3b52d8af420b95f580e32ada5d93 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/C/f34b3b52d8af420b95f580e32ada5d93 2024-12-08T00:15:12,712 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/recovered.edits/354.seqid to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858/recovered.edits/354.seqid 2024-12-08T00:15:12,713 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c31d593296c0be255698232087c08858 2024-12-08T00:15:12,713 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T00:15:12,713 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T00:15:12,714 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T00:15:12,717 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120803b43610eb42460897ad8f3ea6663f4f_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120803b43610eb42460897ad8f3ea6663f4f_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,718 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120828faf857090f4b078f76e64d126c64cf_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120828faf857090f4b078f76e64d126c64cf_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,719 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083de4b52427ce45ef8e35138ad13568d9_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083de4b52427ce45ef8e35138ad13568d9_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,721 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084d6f42c10f2940b4bfa8eff351bf9bbb_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084d6f42c10f2940b4bfa8eff351bf9bbb_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,722 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208635af24af94f4dee9c9ec573d18bfa8a_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208635af24af94f4dee9c9ec573d18bfa8a_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,723 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087b934aaa01c0428dafa1327c896f8a6e_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087b934aaa01c0428dafa1327c896f8a6e_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,724 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089570ea7ec0344c4eb07b3dd285b3ab25_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089570ea7ec0344c4eb07b3dd285b3ab25_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,725 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a44cfbdd43d5498186afc71c71309b23_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a44cfbdd43d5498186afc71c71309b23_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,726 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a54ca42d5f4e4ec4ade5ae99fc219720_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a54ca42d5f4e4ec4ade5ae99fc219720_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,727 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a61c769d4825429cb4cb22baa6e947e7_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a61c769d4825429cb4cb22baa6e947e7_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,728 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208aadda5d5aad64ad3a189175a79a923ff_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208aadda5d5aad64ad3a189175a79a923ff_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,729 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bd99caf9060b4163961d50621dacebf0_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bd99caf9060b4163961d50621dacebf0_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,730 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c92f720e196749f69a6703ba16e62f21_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c92f720e196749f69a6703ba16e62f21_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,731 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c931e6cc486145cea81ce15624fc24d7_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c931e6cc486145cea81ce15624fc24d7_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,732 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e663933101b34de295aec244a07d75e9_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e663933101b34de295aec244a07d75e9_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,733 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ee1aae7b4ee542e7aa7c89692b750cef_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ee1aae7b4ee542e7aa7c89692b750cef_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,734 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f47cd5efe7d8476c8549cac3e1209562_c31d593296c0be255698232087c08858 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f47cd5efe7d8476c8549cac3e1209562_c31d593296c0be255698232087c08858 2024-12-08T00:15:12,734 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T00:15:12,736 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:12,738 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T00:15:12,740 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T00:15:12,741 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:12,741 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T00:15:12,741 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733616912741"}]},"ts":"9223372036854775807"} 2024-12-08T00:15:12,743 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T00:15:12,743 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c31d593296c0be255698232087c08858, NAME => 'TestAcidGuarantees,,1733616877153.c31d593296c0be255698232087c08858.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T00:15:12,743 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T00:15:12,743 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733616912743"}]},"ts":"9223372036854775807"} 2024-12-08T00:15:12,745 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T00:15:12,745 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:12,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-12-08T00:15:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T00:15:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T00:15:12,792 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:12,802 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=353 (was 400), ProcessCount=11 (was 11), AvailableMemoryMB=9282 (was 8350) - AvailableMemoryMB LEAK? - 2024-12-08T00:15:12,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=353, ProcessCount=11, AvailableMemoryMB=9282 2024-12-08T00:15:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T00:15:12,815 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:15:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:12,817 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T00:15:12,818 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:12,818 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-08T00:15:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T00:15:12,818 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T00:15:12,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742032_1208 (size=1061) 2024-12-08T00:15:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T00:15:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T00:15:13,227 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329 2024-12-08T00:15:13,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742033_1209 (size=53) 2024-12-08T00:15:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T00:15:13,634 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:15:13,635 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7cf59c9ae144cfe9fe4bafb7bc62bf46, disabling compactions & flushes 2024-12-08T00:15:13,635 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:13,635 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:13,635 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. after waiting 0 ms 2024-12-08T00:15:13,635 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:13,635 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:13,635 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: Waiting for close lock at 1733616913634Disabling compacts and flushes for region at 1733616913634Disabling writes for close at 1733616913635 (+1 ms)Writing region close event to WAL at 1733616913635Closed at 1733616913635 2024-12-08T00:15:13,637 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T00:15:13,638 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733616913637"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733616913637"}]},"ts":"1733616913637"} 2024-12-08T00:15:13,640 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T00:15:13,642 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T00:15:13,642 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616913642"}]},"ts":"1733616913642"} 2024-12-08T00:15:13,645 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T00:15:13,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cf59c9ae144cfe9fe4bafb7bc62bf46, ASSIGN}] 2024-12-08T00:15:13,647 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cf59c9ae144cfe9fe4bafb7bc62bf46, ASSIGN 2024-12-08T00:15:13,648 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cf59c9ae144cfe9fe4bafb7bc62bf46, ASSIGN; state=OFFLINE, location=c1a8cf8d4cfa,36021,1733616847970; forceNewPlan=false, retain=false 2024-12-08T00:15:13,799 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7cf59c9ae144cfe9fe4bafb7bc62bf46, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:13,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cf59c9ae144cfe9fe4bafb7bc62bf46, ASSIGN because future has completed 2024-12-08T00:15:13,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:15:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T00:15:13,960 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:13,960 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:15:13,960 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,960 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:15:13,960 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,960 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,962 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,963 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:15:13,964 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cf59c9ae144cfe9fe4bafb7bc62bf46 columnFamilyName A 2024-12-08T00:15:13,964 DEBUG [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:13,964 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] regionserver.HStore(327): Store=7cf59c9ae144cfe9fe4bafb7bc62bf46/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:15:13,964 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,966 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:15:13,966 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cf59c9ae144cfe9fe4bafb7bc62bf46 columnFamilyName B 2024-12-08T00:15:13,966 DEBUG [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:13,967 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] regionserver.HStore(327): Store=7cf59c9ae144cfe9fe4bafb7bc62bf46/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:15:13,967 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,968 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:15:13,968 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cf59c9ae144cfe9fe4bafb7bc62bf46 columnFamilyName C 2024-12-08T00:15:13,968 DEBUG [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:13,969 INFO [StoreOpener-7cf59c9ae144cfe9fe4bafb7bc62bf46-1 {}] regionserver.HStore(327): Store=7cf59c9ae144cfe9fe4bafb7bc62bf46/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:15:13,969 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:13,969 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,970 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,970 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,971 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,971 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,972 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:15:13,973 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,974 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T00:15:13,975 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 7cf59c9ae144cfe9fe4bafb7bc62bf46; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67897692, jitterRate=0.011754453182220459}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:15:13,975 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:13,975 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: Running coprocessor pre-open hook at 1733616913960Writing region info on filesystem at 1733616913960Initializing all the Stores at 1733616913961 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616913961Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616913962 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616913962Cleaning up temporary data from old regions at 1733616913971 (+9 ms)Running coprocessor post-open hooks at 1733616913975 (+4 ms)Region opened successfully at 1733616913975 2024-12-08T00:15:13,976 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., pid=61, masterSystemTime=1733616913957 2024-12-08T00:15:13,977 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:13,977 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:13,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7cf59c9ae144cfe9fe4bafb7bc62bf46, regionState=OPEN, openSeqNum=2, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:13,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:15:13,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-08T00:15:13,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 in 177 msec 2024-12-08T00:15:13,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-08T00:15:13,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cf59c9ae144cfe9fe4bafb7bc62bf46, ASSIGN in 336 msec 2024-12-08T00:15:13,983 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T00:15:13,984 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616913983"}]},"ts":"1733616913983"} 2024-12-08T00:15:13,985 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T00:15:13,986 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T00:15:13,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-12-08T00:15:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T00:15:14,961 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:14,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e4e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:14,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:14,964 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6e553d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:14,967 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b89852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,968 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:14,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:14,973 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T00:15:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:15:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@588d8843 2024-12-08T00:15:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T00:15:14,975 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T00:15:14,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:14,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:14,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f48fe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:14,980 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd78348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:14,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:14,982 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce7111b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:14,984 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4718c117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:14,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:14,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3587b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:14,987 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5affdc38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:14,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:14,990 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d8b4a04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:14,992 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2557b895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:14,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:14,994 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9a1ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:14,996 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52078707, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:14,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:14,998 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78a041f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:15,000 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5729ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:15,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:15,002 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121a733b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:15,003 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61373c7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:15,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:15,007 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260bcf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:15,008 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22501971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:15,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:15,012 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6d6ea7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:15,014 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3500aa96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:15,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:15,015 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459b9868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:15,016 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6620b4d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:15,022 DEBUG [Thread-1006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,022 DEBUG [Thread-1004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,022 DEBUG [Thread-993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,022 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,022 DEBUG [Thread-1006 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,023 DEBUG [Thread-1006 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,023 DEBUG [Thread-1010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-08T00:15:15,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T00:15:15,025 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:15,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,025 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:15,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,029 DEBUG [Thread-1001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,030 DEBUG [Thread-997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,030 DEBUG [Thread-995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,031 DEBUG [Thread-1012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,037 DEBUG [Thread-999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:15:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:15,046 DEBUG [Thread-1008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:15,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:15:15,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616915055, max timestamp=1733616915073Num uniques 859; 2024-12-08T00:15:15,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616915055, max timestamp=1733616915073Num uniques 859; 2024-12-08T00:15:15,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:15:15,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:15:15,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616915055, max timestamp=1733616915073Num uniques 859; 2024-12-08T00:15:15,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:15:15,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/de4b43b6e6154084867321603f3f9a8e is 50, key is test_row_0/A:col10/1733616915039/Put/seqid=0 2024-12-08T00:15:15,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616975097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616975099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616975098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616975099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616975103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742034_1210 (size=12001) 2024-12-08T00:15:15,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/de4b43b6e6154084867321603f3f9a8e 2024-12-08T00:15:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T00:15:15,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/6eb773222ed84b26a67508e465358607 is 50, key is test_row_0/B:col10/1733616915039/Put/seqid=0 2024-12-08T00:15:15,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-08T00:15:15,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:15,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:15,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:15,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:15,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:15,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742035_1211 (size=12001) 2024-12-08T00:15:15,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/6eb773222ed84b26a67508e465358607 2024-12-08T00:15:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616975211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616975211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616975211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616975211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616975211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a61d78c0b43a4550acc5baaebbe3812a is 50, key is test_row_0/C:col10/1733616915039/Put/seqid=0 2024-12-08T00:15:15,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742036_1212 (size=12001) 2024-12-08T00:15:15,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a61d78c0b43a4550acc5baaebbe3812a 2024-12-08T00:15:15,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/de4b43b6e6154084867321603f3f9a8e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/de4b43b6e6154084867321603f3f9a8e 2024-12-08T00:15:15,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/de4b43b6e6154084867321603f3f9a8e, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T00:15:15,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/6eb773222ed84b26a67508e465358607 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb773222ed84b26a67508e465358607 2024-12-08T00:15:15,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb773222ed84b26a67508e465358607, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T00:15:15,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a61d78c0b43a4550acc5baaebbe3812a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a61d78c0b43a4550acc5baaebbe3812a 2024-12-08T00:15:15,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a61d78c0b43a4550acc5baaebbe3812a, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T00:15:15,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 233ms, sequenceid=12, compaction requested=false 2024-12-08T00:15:15,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:15,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-08T00:15:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:15,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-08T00:15:15,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:15,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:15,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:15,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:15,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:15,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T00:15:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/075f793d50474882974e52972a636189 is 50, key is test_row_0/A:col10/1733616915101/Put/seqid=0 2024-12-08T00:15:15,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742037_1213 (size=12001) 2024-12-08T00:15:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:15,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:15,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616975455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616975457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616975459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616975460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616975460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616975570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616975570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616975571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616975571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616975571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T00:15:15,748 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/075f793d50474882974e52972a636189 2024-12-08T00:15:15,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/1e90edd2f58c45c29c5d7cacce5f46b3 is 50, key is test_row_0/B:col10/1733616915101/Put/seqid=0 2024-12-08T00:15:15,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742038_1214 (size=12001) 2024-12-08T00:15:15,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616975781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616975781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616975781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616975781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616975781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616976090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616976090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616976091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616976091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616976091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,160 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/1e90edd2f58c45c29c5d7cacce5f46b3 2024-12-08T00:15:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T00:15:16,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/5755ccd49f994531b95ee3480b493680 is 50, key is test_row_0/C:col10/1733616915101/Put/seqid=0 2024-12-08T00:15:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742039_1215 (size=12001) 2024-12-08T00:15:16,574 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/5755ccd49f994531b95ee3480b493680 2024-12-08T00:15:16,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/075f793d50474882974e52972a636189 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/075f793d50474882974e52972a636189 2024-12-08T00:15:16,589 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/075f793d50474882974e52972a636189, entries=150, sequenceid=48, filesize=11.7 K 2024-12-08T00:15:16,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/1e90edd2f58c45c29c5d7cacce5f46b3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e90edd2f58c45c29c5d7cacce5f46b3 2024-12-08T00:15:16,597 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e90edd2f58c45c29c5d7cacce5f46b3, entries=150, sequenceid=48, filesize=11.7 K 2024-12-08T00:15:16,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/5755ccd49f994531b95ee3480b493680 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5755ccd49f994531b95ee3480b493680 2024-12-08T00:15:16,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616976601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616976601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616976601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616976601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616976601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5755ccd49f994531b95ee3480b493680, entries=150, sequenceid=48, filesize=11.7 K 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:16,605 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=67.09 KB/68700 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1267ms, sequenceid=48, compaction requested=false 2024-12-08T00:15:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-08T00:15:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-08T00:15:16,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-08T00:15:16,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-12-08T00:15:16,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.5890 sec 2024-12-08T00:15:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T00:15:17,172 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:17,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-08T00:15:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T00:15:17,175 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:17,176 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:17,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T00:15:17,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T00:15:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:17,331 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:15:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/2eadcf078fad43298efa5ecc6299046d is 50, key is test_row_0/A:col10/1733616915454/Put/seqid=0 2024-12-08T00:15:17,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742040_1216 (size=12001) 2024-12-08T00:15:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T00:15:17,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:17,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:17,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616977636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616977637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616977638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616977639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616977639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,747 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/2eadcf078fad43298efa5ecc6299046d 2024-12-08T00:15:17,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/2d163be4416e45278db9959a1a910b17 is 50, key is test_row_0/B:col10/1733616915454/Put/seqid=0 2024-12-08T00:15:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616977750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616977751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616977751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616977751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616977751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742041_1217 (size=12001) 2024-12-08T00:15:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T00:15:17,932 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T00:15:17,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616977960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616977960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616977961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616977961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:17,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616977961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/2d163be4416e45278db9959a1a910b17 2024-12-08T00:15:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/e8086cf053b3426693514814c4e915d7 is 50, key is test_row_0/C:col10/1733616915454/Put/seqid=0 2024-12-08T00:15:18,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742042_1218 (size=12001) 2024-12-08T00:15:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616978270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616978270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616978270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616978271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616978271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T00:15:18,568 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/e8086cf053b3426693514814c4e915d7 2024-12-08T00:15:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/2eadcf078fad43298efa5ecc6299046d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/2eadcf078fad43298efa5ecc6299046d 2024-12-08T00:15:18,587 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/2eadcf078fad43298efa5ecc6299046d, entries=150, sequenceid=61, filesize=11.7 K 2024-12-08T00:15:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/2d163be4416e45278db9959a1a910b17 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/2d163be4416e45278db9959a1a910b17 2024-12-08T00:15:18,595 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/2d163be4416e45278db9959a1a910b17, entries=150, sequenceid=61, filesize=11.7 K 2024-12-08T00:15:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/e8086cf053b3426693514814c4e915d7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e8086cf053b3426693514814c4e915d7 2024-12-08T00:15:18,605 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e8086cf053b3426693514814c4e915d7, entries=150, sequenceid=61, filesize=11.7 K 2024-12-08T00:15:18,608 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1277ms, sequenceid=61, compaction requested=true 2024-12-08T00:15:18,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:18,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:18,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-08T00:15:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-08T00:15:18,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-08T00:15:18,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-12-08T00:15:18,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4400 sec 2024-12-08T00:15:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:18,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:15:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:18,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/83aa11a7ce21408ca96eddbda6d0cad8 is 50, key is test_row_0/A:col10/1733616917638/Put/seqid=0 2024-12-08T00:15:18,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742043_1219 (size=12001) 2024-12-08T00:15:18,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/83aa11a7ce21408ca96eddbda6d0cad8 2024-12-08T00:15:18,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616978816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616978821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616978821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616978822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616978824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/1e80055cbc1a4e798ab898501e98618d is 50, key is test_row_0/B:col10/1733616917638/Put/seqid=0 2024-12-08T00:15:18,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742044_1220 (size=12001) 2024-12-08T00:15:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616978931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616978931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616978931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616978931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:18,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616978941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616979140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616979141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616979141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616979141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616979151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/1e80055cbc1a4e798ab898501e98618d 2024-12-08T00:15:19,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a8c998cda3a147bdad6ca7cfc38d0fcd is 50, key is test_row_0/C:col10/1733616917638/Put/seqid=0 2024-12-08T00:15:19,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742045_1221 (size=12001) 2024-12-08T00:15:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T00:15:19,321 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:19,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-08T00:15:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T00:15:19,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:19,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:19,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T00:15:19,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616979451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616979451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616979451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616979452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616979460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:19,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T00:15:19,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:19,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:19,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:19,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:19,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:19,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T00:15:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:19,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T00:15:19,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a8c998cda3a147bdad6ca7cfc38d0fcd 2024-12-08T00:15:19,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/83aa11a7ce21408ca96eddbda6d0cad8 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/83aa11a7ce21408ca96eddbda6d0cad8 2024-12-08T00:15:19,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/83aa11a7ce21408ca96eddbda6d0cad8, entries=150, sequenceid=82, filesize=11.7 K 2024-12-08T00:15:19,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/1e80055cbc1a4e798ab898501e98618d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e80055cbc1a4e798ab898501e98618d 2024-12-08T00:15:19,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e80055cbc1a4e798ab898501e98618d, entries=150, sequenceid=82, filesize=11.7 K 2024-12-08T00:15:19,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a8c998cda3a147bdad6ca7cfc38d0fcd as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a8c998cda3a147bdad6ca7cfc38d0fcd 2024-12-08T00:15:19,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a8c998cda3a147bdad6ca7cfc38d0fcd, entries=150, sequenceid=82, filesize=11.7 K 2024-12-08T00:15:19,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 903ms, sequenceid=82, compaction requested=true 2024-12-08T00:15:19,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:19,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:19,687 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:19,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:19,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:19,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:19,687 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:19,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:19,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:19,688 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:19,689 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/A is initiating minor compaction (all files) 2024-12-08T00:15:19,689 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/A in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:19,689 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/de4b43b6e6154084867321603f3f9a8e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/075f793d50474882974e52972a636189, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/2eadcf078fad43298efa5ecc6299046d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/83aa11a7ce21408ca96eddbda6d0cad8] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=46.9 K 2024-12-08T00:15:19,689 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:19,689 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/B is initiating minor compaction (all files) 2024-12-08T00:15:19,689 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/B in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:19,689 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb773222ed84b26a67508e465358607, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e90edd2f58c45c29c5d7cacce5f46b3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/2d163be4416e45278db9959a1a910b17, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e80055cbc1a4e798ab898501e98618d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=46.9 K 2024-12-08T00:15:19,689 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting de4b43b6e6154084867321603f3f9a8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733616915034 2024-12-08T00:15:19,689 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb773222ed84b26a67508e465358607, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733616915034 2024-12-08T00:15:19,690 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e90edd2f58c45c29c5d7cacce5f46b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733616915097 2024-12-08T00:15:19,690 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 075f793d50474882974e52972a636189, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733616915097 2024-12-08T00:15:19,690 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d163be4416e45278db9959a1a910b17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733616915454 2024-12-08T00:15:19,690 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eadcf078fad43298efa5ecc6299046d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733616915454 2024-12-08T00:15:19,690 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e80055cbc1a4e798ab898501e98618d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733616917633 2024-12-08T00:15:19,691 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83aa11a7ce21408ca96eddbda6d0cad8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733616917633 2024-12-08T00:15:19,700 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#A#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:19,701 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/1b72bbcbdcf54195b85b3fc7567e8238 is 50, key is test_row_0/A:col10/1733616917638/Put/seqid=0 2024-12-08T00:15:19,703 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#B#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:19,704 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/d1c6765ebf8b46e2ae04a50d906569bf is 50, key is test_row_0/B:col10/1733616917638/Put/seqid=0 2024-12-08T00:15:19,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742046_1222 (size=12139) 2024-12-08T00:15:19,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742047_1223 (size=12139) 2024-12-08T00:15:19,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T00:15:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:19,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T00:15:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:19,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/a60b5127a4414cc4ad7dc3a05063297a is 50, key is test_row_0/A:col10/1733616918820/Put/seqid=0 2024-12-08T00:15:19,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742048_1224 (size=12001) 2024-12-08T00:15:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T00:15:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:19,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:20,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616980023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616980024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616980024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616980031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616980031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,115 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/1b72bbcbdcf54195b85b3fc7567e8238 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1b72bbcbdcf54195b85b3fc7567e8238 2024-12-08T00:15:20,122 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/A of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 1b72bbcbdcf54195b85b3fc7567e8238(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:20,122 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:20,122 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/A, priority=12, startTime=1733616919687; duration=0sec 2024-12-08T00:15:20,123 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:20,123 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:A 2024-12-08T00:15:20,123 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:20,125 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:20,125 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/C is initiating minor compaction (all files) 2024-12-08T00:15:20,125 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/C in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:20,125 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a61d78c0b43a4550acc5baaebbe3812a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5755ccd49f994531b95ee3480b493680, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e8086cf053b3426693514814c4e915d7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a8c998cda3a147bdad6ca7cfc38d0fcd] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=46.9 K 2024-12-08T00:15:20,126 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting a61d78c0b43a4550acc5baaebbe3812a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733616915034 2024-12-08T00:15:20,126 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5755ccd49f994531b95ee3480b493680, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733616915097 2024-12-08T00:15:20,126 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/d1c6765ebf8b46e2ae04a50d906569bf as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/d1c6765ebf8b46e2ae04a50d906569bf 2024-12-08T00:15:20,126 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8086cf053b3426693514814c4e915d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733616915454 2024-12-08T00:15:20,127 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8c998cda3a147bdad6ca7cfc38d0fcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733616917633 2024-12-08T00:15:20,131 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/B of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into d1c6765ebf8b46e2ae04a50d906569bf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:20,131 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:20,131 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/B, priority=12, startTime=1733616919687; duration=0sec 2024-12-08T00:15:20,131 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:20,131 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:B 2024-12-08T00:15:20,137 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#C#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:20,138 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/e83c70d0d608486c861e2cbf657335a1 is 50, key is test_row_0/C:col10/1733616917638/Put/seqid=0 2024-12-08T00:15:20,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616980141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616980142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616980142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616980142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616980142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742049_1225 (size=12139) 2024-12-08T00:15:20,190 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/a60b5127a4414cc4ad7dc3a05063297a 2024-12-08T00:15:20,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/bbbc16f8b4014f0fb40428d005f6db5d is 50, key is test_row_0/B:col10/1733616918820/Put/seqid=0 2024-12-08T00:15:20,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742050_1226 (size=12001) 2024-12-08T00:15:20,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616980350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616980351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616980351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616980351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616980351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T00:15:20,554 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/e83c70d0d608486c861e2cbf657335a1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e83c70d0d608486c861e2cbf657335a1 2024-12-08T00:15:20,560 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/C of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into e83c70d0d608486c861e2cbf657335a1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:20,560 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:20,561 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/C, priority=12, startTime=1733616919687; duration=0sec 2024-12-08T00:15:20,561 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:20,561 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:C 2024-12-08T00:15:20,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/bbbc16f8b4014f0fb40428d005f6db5d 2024-12-08T00:15:20,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/1a2a60c3f30b41d49617f70308f8def3 is 50, key is test_row_0/C:col10/1733616918820/Put/seqid=0 2024-12-08T00:15:20,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742051_1227 (size=12001) 2024-12-08T00:15:20,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616980660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616980660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616980661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616980661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616980661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,015 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/1a2a60c3f30b41d49617f70308f8def3 2024-12-08T00:15:21,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/a60b5127a4414cc4ad7dc3a05063297a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a60b5127a4414cc4ad7dc3a05063297a 2024-12-08T00:15:21,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a60b5127a4414cc4ad7dc3a05063297a, entries=150, sequenceid=94, filesize=11.7 K 2024-12-08T00:15:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/bbbc16f8b4014f0fb40428d005f6db5d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/bbbc16f8b4014f0fb40428d005f6db5d 2024-12-08T00:15:21,029 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/bbbc16f8b4014f0fb40428d005f6db5d, entries=150, sequenceid=94, filesize=11.7 K 2024-12-08T00:15:21,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/1a2a60c3f30b41d49617f70308f8def3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1a2a60c3f30b41d49617f70308f8def3 2024-12-08T00:15:21,034 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1a2a60c3f30b41d49617f70308f8def3, entries=150, sequenceid=94, filesize=11.7 K 2024-12-08T00:15:21,035 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1253ms, sequenceid=94, compaction requested=false 2024-12-08T00:15:21,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:21,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:21,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-08T00:15:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-08T00:15:21,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-08T00:15:21,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7130 sec 2024-12-08T00:15:21,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7170 sec 2024-12-08T00:15:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:21,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616920023, max timestamp=1733616921172Num uniques 859; 2024-12-08T00:15:21,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:15:21,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616920023, max timestamp=1733616921172Num uniques 859; 2024-12-08T00:15:21,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616920023, max timestamp=1733616921172Num uniques 859; 2024-12-08T00:15:21,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:15:21,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:15:21,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=46.96 KB heapSize=126 KB 2024-12-08T00:15:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:21,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/b7654030927d4e8cb387c84fccb5c61f is 50, key is test_row_0/A:col10/1733616921174/Put/seqid=0 2024-12-08T00:15:21,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742052_1228 (size=12001) 2024-12-08T00:15:21,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/b7654030927d4e8cb387c84fccb5c61f 2024-12-08T00:15:21,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/430b070feb194930adec7b1ce080ebac is 50, key is test_row_0/B:col10/1733616921174/Put/seqid=0 2024-12-08T00:15:21,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733616921190, max timestamp=1733616921198Num uniques 859; 2024-12-08T00:15:21,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T00:15:21,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733616921190, max timestamp=1733616921198Num uniques 859; 2024-12-08T00:15:21,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733616921190, max timestamp=1733616921198Num uniques 859; 2024-12-08T00:15:21,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T00:15:21,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T00:15:21,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742053_1229 (size=12001) 2024-12-08T00:15:21,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/430b070feb194930adec7b1ce080ebac 2024-12-08T00:15:21,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616981215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616981216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616981218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616981218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616981220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/2dc6be93e2794fe9b00fdad32955af30 is 50, key is test_row_0/C:col10/1733616921174/Put/seqid=0 2024-12-08T00:15:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742054_1230 (size=12001) 2024-12-08T00:15:21,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/2dc6be93e2794fe9b00fdad32955af30 2024-12-08T00:15:21,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/b7654030927d4e8cb387c84fccb5c61f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b7654030927d4e8cb387c84fccb5c61f 2024-12-08T00:15:21,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b7654030927d4e8cb387c84fccb5c61f, entries=150, sequenceid=122, filesize=11.7 K 2024-12-08T00:15:21,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/430b070feb194930adec7b1ce080ebac as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/430b070feb194930adec7b1ce080ebac 2024-12-08T00:15:21,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/430b070feb194930adec7b1ce080ebac, entries=150, sequenceid=122, filesize=11.7 K 2024-12-08T00:15:21,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/2dc6be93e2794fe9b00fdad32955af30 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/2dc6be93e2794fe9b00fdad32955af30 2024-12-08T00:15:21,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/2dc6be93e2794fe9b00fdad32955af30, entries=150, sequenceid=122, filesize=11.7 K 2024-12-08T00:15:21,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 77ms, sequenceid=122, compaction requested=true 2024-12-08T00:15:21,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:21,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:21,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:21,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:21,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:21,251 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:21,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:21,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T00:15:21,252 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:21,253 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:21,253 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/B is initiating minor compaction (all files) 2024-12-08T00:15:21,253 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/B in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:21,253 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/d1c6765ebf8b46e2ae04a50d906569bf, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/bbbc16f8b4014f0fb40428d005f6db5d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/430b070feb194930adec7b1ce080ebac] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=35.3 K 2024-12-08T00:15:21,253 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:21,253 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c6765ebf8b46e2ae04a50d906569bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733616917633 2024-12-08T00:15:21,253 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/A is initiating minor compaction (all files) 2024-12-08T00:15:21,254 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/A in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:21,254 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1b72bbcbdcf54195b85b3fc7567e8238, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a60b5127a4414cc4ad7dc3a05063297a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b7654030927d4e8cb387c84fccb5c61f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=35.3 K 2024-12-08T00:15:21,254 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting bbbc16f8b4014f0fb40428d005f6db5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733616918819 2024-12-08T00:15:21,254 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b72bbcbdcf54195b85b3fc7567e8238, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733616917633 2024-12-08T00:15:21,254 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 430b070feb194930adec7b1ce080ebac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733616921172 2024-12-08T00:15:21,254 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting a60b5127a4414cc4ad7dc3a05063297a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733616918819 2024-12-08T00:15:21,255 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7654030927d4e8cb387c84fccb5c61f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733616921172 2024-12-08T00:15:21,262 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:21,263 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/f45df10247cf4803bf045419fbc01449 is 50, key is test_row_0/B:col10/1733616921174/Put/seqid=0 2024-12-08T00:15:21,265 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#A#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:21,265 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/c3cf570af7a3483ab82327114f84eaf9 is 50, key is test_row_0/A:col10/1733616921174/Put/seqid=0 2024-12-08T00:15:21,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742055_1231 (size=12241) 2024-12-08T00:15:21,279 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/f45df10247cf4803bf045419fbc01449 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f45df10247cf4803bf045419fbc01449 2024-12-08T00:15:21,286 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/B of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into f45df10247cf4803bf045419fbc01449(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:21,286 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:21,286 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/B, priority=13, startTime=1733616921251; duration=0sec 2024-12-08T00:15:21,286 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:21,286 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:B 2024-12-08T00:15:21,286 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:21,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742056_1232 (size=12241) 2024-12-08T00:15:21,288 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:21,289 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/C is initiating minor compaction (all files) 2024-12-08T00:15:21,289 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/C in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:21,289 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e83c70d0d608486c861e2cbf657335a1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1a2a60c3f30b41d49617f70308f8def3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/2dc6be93e2794fe9b00fdad32955af30] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=35.3 K 2024-12-08T00:15:21,290 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e83c70d0d608486c861e2cbf657335a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733616917633 2024-12-08T00:15:21,290 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a2a60c3f30b41d49617f70308f8def3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733616918819 2024-12-08T00:15:21,291 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dc6be93e2794fe9b00fdad32955af30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733616921172 2024-12-08T00:15:21,300 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#C#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:21,301 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/1ce5a2da26b048c489bdbdde9413d4a0 is 50, key is test_row_0/C:col10/1733616921174/Put/seqid=0 2024-12-08T00:15:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742057_1233 (size=12241) 2024-12-08T00:15:21,329 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/1ce5a2da26b048c489bdbdde9413d4a0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1ce5a2da26b048c489bdbdde9413d4a0 2024-12-08T00:15:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:21,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-08T00:15:21,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:21,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:21,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:21,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:21,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:21,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:21,338 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/C of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 1ce5a2da26b048c489bdbdde9413d4a0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:21,338 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:21,338 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/C, priority=13, startTime=1733616921251; duration=0sec 2024-12-08T00:15:21,338 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:21,338 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:C 2024-12-08T00:15:21,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/ab2363a22a414103953ba32f4531d0f0 is 50, key is test_row_0/A:col10/1733616921216/Put/seqid=0 2024-12-08T00:15:21,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616981337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616981338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742058_1234 (size=12151) 2024-12-08T00:15:21,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/ab2363a22a414103953ba32f4531d0f0 2024-12-08T00:15:21,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616981343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616981344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616981344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/47b419653e6d47a2b4ea487241951e52 is 50, key is test_row_0/B:col10/1733616921216/Put/seqid=0 2024-12-08T00:15:21,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742059_1235 (size=12151) 2024-12-08T00:15:21,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/47b419653e6d47a2b4ea487241951e52 2024-12-08T00:15:21,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/23c10d6086c144bdb3deb5125d78818e is 50, key is test_row_0/C:col10/1733616921216/Put/seqid=0 2024-12-08T00:15:21,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742060_1236 (size=12151) 2024-12-08T00:15:21,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616981451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616981451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616981461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616981461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616981461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T00:15:21,471 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:21,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-08T00:15:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T00:15:21,473 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:21,473 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:21,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T00:15:21,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T00:15:21,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:21,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:21,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:21,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:21,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:21,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616981661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616981661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616981671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616981671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616981671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:21,695 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/c3cf570af7a3483ab82327114f84eaf9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/c3cf570af7a3483ab82327114f84eaf9 2024-12-08T00:15:21,702 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/A of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into c3cf570af7a3483ab82327114f84eaf9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:21,702 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:21,702 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/A, priority=13, startTime=1733616921251; duration=0sec 2024-12-08T00:15:21,702 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:21,702 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:A 2024-12-08T00:15:21,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T00:15:21,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:21,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:21,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:21,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:21,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T00:15:21,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/23c10d6086c144bdb3deb5125d78818e 2024-12-08T00:15:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/ab2363a22a414103953ba32f4531d0f0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ab2363a22a414103953ba32f4531d0f0 2024-12-08T00:15:21,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ab2363a22a414103953ba32f4531d0f0, entries=150, sequenceid=158, filesize=11.9 K 2024-12-08T00:15:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/47b419653e6d47a2b4ea487241951e52 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/47b419653e6d47a2b4ea487241951e52 2024-12-08T00:15:21,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/47b419653e6d47a2b4ea487241951e52, entries=150, sequenceid=158, filesize=11.9 K 2024-12-08T00:15:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/23c10d6086c144bdb3deb5125d78818e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/23c10d6086c144bdb3deb5125d78818e 2024-12-08T00:15:21,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/23c10d6086c144bdb3deb5125d78818e, entries=150, sequenceid=158, filesize=11.9 K 2024-12-08T00:15:21,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 478ms, sequenceid=158, compaction requested=false 2024-12-08T00:15:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:21,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T00:15:21,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:21,932 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:15:21,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:21,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:21,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:21,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:21,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:21,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:21,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/1e53d20f0200457b83200637461bde07 is 50, key is test_row_0/A:col10/1733616921342/Put/seqid=0 2024-12-08T00:15:21,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742061_1237 (size=12151) 2024-12-08T00:15:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:21,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:21,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616921988, max timestamp=1733616921992Num uniques 859; 2024-12-08T00:15:21,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616921988, max timestamp=1733616921992Num uniques 859; 2024-12-08T00:15:21,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:15:21,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:15:21,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616921988, max timestamp=1733616921992Num uniques 859; 2024-12-08T00:15:21,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:15:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616982012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616982012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616982013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616982017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616982017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T00:15:22,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616982121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616982121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616982121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616982133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616982133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616982330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616982330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616982331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616982341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616982341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,345 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/1e53d20f0200457b83200637461bde07 2024-12-08T00:15:22,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/02c990626d1c4c919a32001efcc3cde7 is 50, key is test_row_0/B:col10/1733616921342/Put/seqid=0 2024-12-08T00:15:22,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742062_1238 (size=12151) 2024-12-08T00:15:22,356 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/02c990626d1c4c919a32001efcc3cde7 2024-12-08T00:15:22,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/cdd83ac848524b94bcd32a6c65d53ca9 is 50, key is test_row_0/C:col10/1733616921342/Put/seqid=0 2024-12-08T00:15:22,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742063_1239 (size=12151) 2024-12-08T00:15:22,371 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/cdd83ac848524b94bcd32a6c65d53ca9 2024-12-08T00:15:22,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/1e53d20f0200457b83200637461bde07 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1e53d20f0200457b83200637461bde07 2024-12-08T00:15:22,382 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1e53d20f0200457b83200637461bde07, entries=150, sequenceid=170, filesize=11.9 K 2024-12-08T00:15:22,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/02c990626d1c4c919a32001efcc3cde7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02c990626d1c4c919a32001efcc3cde7 2024-12-08T00:15:22,386 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02c990626d1c4c919a32001efcc3cde7, entries=150, sequenceid=170, filesize=11.9 K 2024-12-08T00:15:22,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/cdd83ac848524b94bcd32a6c65d53ca9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cdd83ac848524b94bcd32a6c65d53ca9 2024-12-08T00:15:22,391 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cdd83ac848524b94bcd32a6c65d53ca9, entries=150, sequenceid=170, filesize=11.9 K 2024-12-08T00:15:22,393 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 461ms, sequenceid=170, compaction requested=true 2024-12-08T00:15:22,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:22,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:22,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-08T00:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-08T00:15:22,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-08T00:15:22,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 921 msec 2024-12-08T00:15:22,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 925 msec 2024-12-08T00:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T00:15:22,611 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:22,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-08T00:15:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T00:15:22,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:22,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:22,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:22,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-08T00:15:22,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733616922009, max timestamp=1733616922642Num uniques 1068; 2024-12-08T00:15:22,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733616922009, max timestamp=1733616922642Num uniques 1068; 2024-12-08T00:15:22,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T00:15:22,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T00:15:22,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733616922009, max timestamp=1733616922642Num uniques 1068; 2024-12-08T00:15:22,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:22,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T00:15:22,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:22,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:22,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:22,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:22,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/46e2cd5ed52347fcb0b107465f019913 is 50, key is test_row_0/A:col10/1733616922643/Put/seqid=0 2024-12-08T00:15:22,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742064_1240 (size=16931) 2024-12-08T00:15:22,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733616922661, max timestamp=1733616922669Num uniques 859; 2024-12-08T00:15:22,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T00:15:22,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733616922661, max timestamp=1733616922669Num uniques 859; 2024-12-08T00:15:22,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T00:15:22,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733616922661, max timestamp=1733616922669Num uniques 859; 2024-12-08T00:15:22,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T00:15:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616982685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616982686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616982687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616982690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616982693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T00:15:22,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T00:15:22,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:22,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:22,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:22,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:22,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:22,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616982801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616982801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616982801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616982801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616982801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:22,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T00:15:22,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:22,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:22,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:22,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:22,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T00:15:23,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616983011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616983011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616983011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616983011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616983011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/46e2cd5ed52347fcb0b107465f019913 2024-12-08T00:15:23,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/6eb6a36deba94b1d94d49292797b09f3 is 50, key is test_row_0/B:col10/1733616922643/Put/seqid=0 2024-12-08T00:15:23,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T00:15:23,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:23,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:23,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:23,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742065_1241 (size=12151) 2024-12-08T00:15:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:23,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/6eb6a36deba94b1d94d49292797b09f3 2024-12-08T00:15:23,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/8c86807b8bba4ef7a63600573ac060e7 is 50, key is test_row_0/C:col10/1733616922643/Put/seqid=0 2024-12-08T00:15:23,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742066_1242 (size=12151) 2024-12-08T00:15:23,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T00:15:23,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:23,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:23,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T00:15:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616983321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616983322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616983322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616983322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616983323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T00:15:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:23,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/8c86807b8bba4ef7a63600573ac060e7 2024-12-08T00:15:23,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/46e2cd5ed52347fcb0b107465f019913 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/46e2cd5ed52347fcb0b107465f019913 2024-12-08T00:15:23,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/46e2cd5ed52347fcb0b107465f019913, entries=250, sequenceid=210, filesize=16.5 K 2024-12-08T00:15:23,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/6eb6a36deba94b1d94d49292797b09f3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb6a36deba94b1d94d49292797b09f3 2024-12-08T00:15:23,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb6a36deba94b1d94d49292797b09f3, entries=150, sequenceid=210, filesize=11.9 K 2024-12-08T00:15:23,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/8c86807b8bba4ef7a63600573ac060e7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8c86807b8bba4ef7a63600573ac060e7 2024-12-08T00:15:23,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8c86807b8bba4ef7a63600573ac060e7, entries=150, sequenceid=210, filesize=11.9 K 2024-12-08T00:15:23,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=147.60 KB/151140 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 876ms, sequenceid=210, compaction requested=true 2024-12-08T00:15:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:23,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:23,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:23,520 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:23,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:23,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:23,520 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:23,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:23,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:23,521 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:23,521 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:23,521 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/A is initiating minor compaction (all files) 2024-12-08T00:15:23,521 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/B is initiating minor compaction (all files) 2024-12-08T00:15:23,521 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/A in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,521 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/B in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,521 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/c3cf570af7a3483ab82327114f84eaf9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ab2363a22a414103953ba32f4531d0f0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1e53d20f0200457b83200637461bde07, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/46e2cd5ed52347fcb0b107465f019913] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=52.2 K 2024-12-08T00:15:23,521 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f45df10247cf4803bf045419fbc01449, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/47b419653e6d47a2b4ea487241951e52, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02c990626d1c4c919a32001efcc3cde7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb6a36deba94b1d94d49292797b09f3] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=47.6 K 2024-12-08T00:15:23,521 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting f45df10247cf4803bf045419fbc01449, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733616921172 2024-12-08T00:15:23,521 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3cf570af7a3483ab82327114f84eaf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733616921172 2024-12-08T00:15:23,521 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab2363a22a414103953ba32f4531d0f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733616921205 2024-12-08T00:15:23,521 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 47b419653e6d47a2b4ea487241951e52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733616921205 2024-12-08T00:15:23,522 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e53d20f0200457b83200637461bde07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733616921340 2024-12-08T00:15:23,522 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c990626d1c4c919a32001efcc3cde7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733616921340 2024-12-08T00:15:23,522 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e2cd5ed52347fcb0b107465f019913, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733616922009 2024-12-08T00:15:23,522 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb6a36deba94b1d94d49292797b09f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733616922016 2024-12-08T00:15:23,531 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#B#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:23,531 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:23,532 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/02a3dc647070416eacfd06c4a175749b is 50, key is test_row_0/B:col10/1733616922643/Put/seqid=0 2024-12-08T00:15:23,532 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/57be3ef3e5e44b1e87f37ba0fd705b32 is 50, key is test_row_0/A:col10/1733616922643/Put/seqid=0 2024-12-08T00:15:23,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T00:15:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,533 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-08T00:15:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:23,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742067_1243 (size=12527) 2024-12-08T00:15:23,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/a8ee7e4512aa4f83a0b0c6c6355edf9c is 50, key is test_row_0/A:col10/1733616922685/Put/seqid=0 2024-12-08T00:15:23,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/57be3ef3e5e44b1e87f37ba0fd705b32 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/57be3ef3e5e44b1e87f37ba0fd705b32 2024-12-08T00:15:23,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742068_1244 (size=12527) 2024-12-08T00:15:23,585 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/A of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 57be3ef3e5e44b1e87f37ba0fd705b32(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:23,585 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:23,586 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/A, priority=12, startTime=1733616923519; duration=0sec 2024-12-08T00:15:23,586 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:23,586 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:A 2024-12-08T00:15:23,586 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:23,588 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:23,588 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/C is initiating minor compaction (all files) 2024-12-08T00:15:23,588 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/C in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:23,589 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1ce5a2da26b048c489bdbdde9413d4a0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/23c10d6086c144bdb3deb5125d78818e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cdd83ac848524b94bcd32a6c65d53ca9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8c86807b8bba4ef7a63600573ac060e7] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=47.6 K 2024-12-08T00:15:23,589 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce5a2da26b048c489bdbdde9413d4a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733616921172 2024-12-08T00:15:23,589 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23c10d6086c144bdb3deb5125d78818e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733616921205 2024-12-08T00:15:23,590 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdd83ac848524b94bcd32a6c65d53ca9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733616921340 2024-12-08T00:15:23,590 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c86807b8bba4ef7a63600573ac060e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733616922016 2024-12-08T00:15:23,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742069_1245 (size=12151) 2024-12-08T00:15:23,605 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#C#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:23,606 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/8f02732460ac4f108475a0e4b8786a69 is 50, key is test_row_0/C:col10/1733616922643/Put/seqid=0 2024-12-08T00:15:23,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742070_1246 (size=12527) 2024-12-08T00:15:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T00:15:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:23,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:23,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616983842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616983836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616983843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616983850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616983851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616983960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616983960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616983961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616983961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616983961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:23,992 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/02a3dc647070416eacfd06c4a175749b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02a3dc647070416eacfd06c4a175749b 2024-12-08T00:15:23,999 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/B of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 02a3dc647070416eacfd06c4a175749b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:23,999 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/a8ee7e4512aa4f83a0b0c6c6355edf9c 2024-12-08T00:15:23,999 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:24,000 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/B, priority=12, startTime=1733616923520; duration=0sec 2024-12-08T00:15:24,000 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:24,000 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:B 2024-12-08T00:15:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/8c594e26c08144cb8d7905e74bfd1864 is 50, key is test_row_0/B:col10/1733616922685/Put/seqid=0 2024-12-08T00:15:24,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742071_1247 (size=12151) 2024-12-08T00:15:24,020 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/8f02732460ac4f108475a0e4b8786a69 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8f02732460ac4f108475a0e4b8786a69 2024-12-08T00:15:24,027 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/C of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 8f02732460ac4f108475a0e4b8786a69(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:24,027 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:24,027 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/C, priority=12, startTime=1733616923520; duration=0sec 2024-12-08T00:15:24,027 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:24,027 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:C 2024-12-08T00:15:24,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616984171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616984171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616984171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616984171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616984171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,414 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/8c594e26c08144cb8d7905e74bfd1864 2024-12-08T00:15:24,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/9e868d1fb1254962bb69be9c59ff8357 is 50, key is test_row_0/C:col10/1733616922685/Put/seqid=0 2024-12-08T00:15:24,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742072_1248 (size=12151) 2024-12-08T00:15:24,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616984481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616984481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616984482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616984482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616984482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T00:15:24,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/9e868d1fb1254962bb69be9c59ff8357 2024-12-08T00:15:24,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/a8ee7e4512aa4f83a0b0c6c6355edf9c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a8ee7e4512aa4f83a0b0c6c6355edf9c 2024-12-08T00:15:24,833 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a8ee7e4512aa4f83a0b0c6c6355edf9c, entries=150, sequenceid=246, filesize=11.9 K 2024-12-08T00:15:24,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/8c594e26c08144cb8d7905e74bfd1864 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8c594e26c08144cb8d7905e74bfd1864 2024-12-08T00:15:24,838 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8c594e26c08144cb8d7905e74bfd1864, entries=150, sequenceid=246, filesize=11.9 K 2024-12-08T00:15:24,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/9e868d1fb1254962bb69be9c59ff8357 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/9e868d1fb1254962bb69be9c59ff8357 2024-12-08T00:15:24,843 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/9e868d1fb1254962bb69be9c59ff8357, entries=150, sequenceid=246, filesize=11.9 K 2024-12-08T00:15:24,844 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1311ms, sequenceid=246, compaction requested=false 2024-12-08T00:15:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-08T00:15:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-08T00:15:24,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-08T00:15:24,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2320 sec 2024-12-08T00:15:24,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2360 sec 2024-12-08T00:15:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:24,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T00:15:24,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:24,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:24,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:24,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:24,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:24,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:24,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/683940345d0b4e8492f45a773f9ff6b9 is 50, key is test_row_0/A:col10/1733616924993/Put/seqid=0 2024-12-08T00:15:25,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742073_1249 (size=12251) 2024-12-08T00:15:25,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/683940345d0b4e8492f45a773f9ff6b9 2024-12-08T00:15:25,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/f5c93e7b19364875934bda610f54d254 is 50, key is test_row_0/B:col10/1733616924993/Put/seqid=0 2024-12-08T00:15:25,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616985008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616985011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616985015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616985014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616985015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742074_1250 (size=12251) 2024-12-08T00:15:25,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616985121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616985121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616985131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616985131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616985131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616985331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616985331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616985341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616985341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616985341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/f5c93e7b19364875934bda610f54d254 2024-12-08T00:15:25,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/b82df1a4602e44a0b2a70262d2535ef7 is 50, key is test_row_0/C:col10/1733616924993/Put/seqid=0 2024-12-08T00:15:25,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742075_1251 (size=12251) 2024-12-08T00:15:25,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/b82df1a4602e44a0b2a70262d2535ef7 2024-12-08T00:15:25,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/683940345d0b4e8492f45a773f9ff6b9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/683940345d0b4e8492f45a773f9ff6b9 2024-12-08T00:15:25,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/683940345d0b4e8492f45a773f9ff6b9, entries=150, sequenceid=262, filesize=12.0 K 2024-12-08T00:15:25,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/f5c93e7b19364875934bda610f54d254 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f5c93e7b19364875934bda610f54d254 2024-12-08T00:15:25,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f5c93e7b19364875934bda610f54d254, entries=150, sequenceid=262, filesize=12.0 K 2024-12-08T00:15:25,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/b82df1a4602e44a0b2a70262d2535ef7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b82df1a4602e44a0b2a70262d2535ef7 2024-12-08T00:15:25,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b82df1a4602e44a0b2a70262d2535ef7, entries=150, sequenceid=262, filesize=12.0 K 2024-12-08T00:15:25,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 465ms, sequenceid=262, compaction requested=true 2024-12-08T00:15:25,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:25,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:25,458 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:25,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:25,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:25,458 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:25,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:25,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:25,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:25,459 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:25,459 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:25,459 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/B is initiating minor compaction (all files) 2024-12-08T00:15:25,459 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/A is initiating minor compaction (all files) 2024-12-08T00:15:25,459 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/B in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:25,459 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/A in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:25,459 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02a3dc647070416eacfd06c4a175749b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8c594e26c08144cb8d7905e74bfd1864, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f5c93e7b19364875934bda610f54d254] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=36.1 K 2024-12-08T00:15:25,459 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/57be3ef3e5e44b1e87f37ba0fd705b32, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a8ee7e4512aa4f83a0b0c6c6355edf9c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/683940345d0b4e8492f45a773f9ff6b9] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=36.1 K 2024-12-08T00:15:25,460 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 02a3dc647070416eacfd06c4a175749b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733616922016 2024-12-08T00:15:25,460 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57be3ef3e5e44b1e87f37ba0fd705b32, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733616922016 2024-12-08T00:15:25,460 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c594e26c08144cb8d7905e74bfd1864, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733616922685 2024-12-08T00:15:25,460 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ee7e4512aa4f83a0b0c6c6355edf9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733616922685 2024-12-08T00:15:25,460 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 683940345d0b4e8492f45a773f9ff6b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733616923848 2024-12-08T00:15:25,460 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c93e7b19364875934bda610f54d254, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733616923848 2024-12-08T00:15:25,479 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:25,479 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/3de174aab21f451e9eafbbf16ce2e573 is 50, key is test_row_0/A:col10/1733616924993/Put/seqid=0 2024-12-08T00:15:25,482 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:25,482 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/c2789acb41df459e91769c7f890c116d is 50, key is test_row_0/B:col10/1733616924993/Put/seqid=0 2024-12-08T00:15:25,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742077_1253 (size=12729) 2024-12-08T00:15:25,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742076_1252 (size=12729) 2024-12-08T00:15:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:25,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:15:25,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:25,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:25,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:25,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:25,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:25,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:25,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/7f76ff5e8e7b4d17a393e797da297c4c is 50, key is test_row_0/A:col10/1733616925642/Put/seqid=0 2024-12-08T00:15:25,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742078_1254 (size=12301) 2024-12-08T00:15:25,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616985655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616985656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616985657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616985658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616985658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616985761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616985771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616985771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616985771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616985771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,890 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/3de174aab21f451e9eafbbf16ce2e573 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/3de174aab21f451e9eafbbf16ce2e573 2024-12-08T00:15:25,890 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/c2789acb41df459e91769c7f890c116d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c2789acb41df459e91769c7f890c116d 2024-12-08T00:15:25,895 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/B of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into c2789acb41df459e91769c7f890c116d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:25,895 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/A of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 3de174aab21f451e9eafbbf16ce2e573(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:25,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:25,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:25,895 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/A, priority=13, startTime=1733616925458; duration=0sec 2024-12-08T00:15:25,895 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/B, priority=13, startTime=1733616925458; duration=0sec 2024-12-08T00:15:25,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:25,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:A 2024-12-08T00:15:25,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:25,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:B 2024-12-08T00:15:25,895 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:25,896 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:25,896 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/C is initiating minor compaction (all files) 2024-12-08T00:15:25,896 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/C in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:25,896 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8f02732460ac4f108475a0e4b8786a69, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/9e868d1fb1254962bb69be9c59ff8357, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b82df1a4602e44a0b2a70262d2535ef7] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=36.1 K 2024-12-08T00:15:25,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f02732460ac4f108475a0e4b8786a69, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733616922016 2024-12-08T00:15:25,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e868d1fb1254962bb69be9c59ff8357, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733616922685 2024-12-08T00:15:25,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting b82df1a4602e44a0b2a70262d2535ef7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733616923848 2024-12-08T00:15:25,903 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#C#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:25,904 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/45a39b5298054a82b688ed717719d342 is 50, key is test_row_0/C:col10/1733616924993/Put/seqid=0 2024-12-08T00:15:25,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742079_1255 (size=12729) 2024-12-08T00:15:25,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616985971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616985981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616985981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616985981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616985981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/7f76ff5e8e7b4d17a393e797da297c4c 2024-12-08T00:15:26,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/4a3495f933ff4e14b0af5b7d98442f1f is 50, key is test_row_0/B:col10/1733616925642/Put/seqid=0 2024-12-08T00:15:26,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742080_1256 (size=12301) 2024-12-08T00:15:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616986281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616986291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616986291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616986291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616986291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,323 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/45a39b5298054a82b688ed717719d342 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/45a39b5298054a82b688ed717719d342 2024-12-08T00:15:26,329 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/C of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 45a39b5298054a82b688ed717719d342(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:26,329 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:26,329 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/C, priority=13, startTime=1733616925458; duration=0sec 2024-12-08T00:15:26,329 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:26,329 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:C 2024-12-08T00:15:26,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/4a3495f933ff4e14b0af5b7d98442f1f 2024-12-08T00:15:26,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/69901d4ae3844bc0a82322fcfadc53cd is 50, key is test_row_0/C:col10/1733616925642/Put/seqid=0 2024-12-08T00:15:26,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742081_1257 (size=12301) 2024-12-08T00:15:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T00:15:26,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:26,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-08T00:15:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T00:15:26,773 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:26,773 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:26,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:26,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616986791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616986801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616986801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616986801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616986801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T00:15:26,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T00:15:26,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:26,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:26,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:26,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:26,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:26,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/69901d4ae3844bc0a82322fcfadc53cd 2024-12-08T00:15:26,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/7f76ff5e8e7b4d17a393e797da297c4c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7f76ff5e8e7b4d17a393e797da297c4c 2024-12-08T00:15:26,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7f76ff5e8e7b4d17a393e797da297c4c, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T00:15:26,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/4a3495f933ff4e14b0af5b7d98442f1f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/4a3495f933ff4e14b0af5b7d98442f1f 2024-12-08T00:15:26,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/4a3495f933ff4e14b0af5b7d98442f1f, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T00:15:26,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/69901d4ae3844bc0a82322fcfadc53cd as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/69901d4ae3844bc0a82322fcfadc53cd 2024-12-08T00:15:26,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/69901d4ae3844bc0a82322fcfadc53cd, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T00:15:26,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1304ms, sequenceid=283, compaction requested=false 2024-12-08T00:15:26,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:27,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T00:15:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:27,080 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T00:15:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:27,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/7269e306efe14e1c938a8093473454f5 is 50, key is test_row_0/A:col10/1733616925657/Put/seqid=0 2024-12-08T00:15:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T00:15:27,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742082_1258 (size=12301) 2024-12-08T00:15:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T00:15:27,494 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/7269e306efe14e1c938a8093473454f5 2024-12-08T00:15:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/73563e83dd9e46b4a841295ee23bfeb8 is 50, key is test_row_0/B:col10/1733616925657/Put/seqid=0 2024-12-08T00:15:27,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742083_1259 (size=12301) 2024-12-08T00:15:27,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/73563e83dd9e46b4a841295ee23bfeb8 2024-12-08T00:15:27,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/115cceaaf4114ef785378c8d15737d1d is 50, key is test_row_0/C:col10/1733616925657/Put/seqid=0 2024-12-08T00:15:27,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742084_1260 (size=12301) 2024-12-08T00:15:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:27,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:27,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733616927828, max timestamp=1733616927847Num uniques 859; 2024-12-08T00:15:27,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:15:27,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733616927828, max timestamp=1733616927847Num uniques 859; 2024-12-08T00:15:27,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:15:27,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616927828, max timestamp=1733616927849Num uniques 859; 2024-12-08T00:15:27,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:15:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616987871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616987871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616987874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:27,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616987876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616987878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T00:15:27,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616987981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616987981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:27,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616987991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616987991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:27,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616987991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,003 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/115cceaaf4114ef785378c8d15737d1d 2024-12-08T00:15:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/7269e306efe14e1c938a8093473454f5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7269e306efe14e1c938a8093473454f5 2024-12-08T00:15:28,019 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7269e306efe14e1c938a8093473454f5, entries=150, sequenceid=297, filesize=12.0 K 2024-12-08T00:15:28,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/73563e83dd9e46b4a841295ee23bfeb8 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/73563e83dd9e46b4a841295ee23bfeb8 2024-12-08T00:15:28,032 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/73563e83dd9e46b4a841295ee23bfeb8, entries=150, sequenceid=297, filesize=12.0 K 2024-12-08T00:15:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/115cceaaf4114ef785378c8d15737d1d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/115cceaaf4114ef785378c8d15737d1d 2024-12-08T00:15:28,050 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/115cceaaf4114ef785378c8d15737d1d, entries=150, sequenceid=297, filesize=12.0 K 2024-12-08T00:15:28,055 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 975ms, sequenceid=297, compaction requested=true 2024-12-08T00:15:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-08T00:15:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-08T00:15:28,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-08T00:15:28,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2850 sec 2024-12-08T00:15:28,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.2910 sec 2024-12-08T00:15:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:28,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T00:15:28,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:28,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:28,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:28,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:28,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:28,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:28,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/a61760338d9f410ba822bf82b84ed514 is 50, key is test_row_0/A:col10/1733616927867/Put/seqid=0 2024-12-08T00:15:28,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616988209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616988209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616988211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616988211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616988213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742085_1261 (size=12301) 2024-12-08T00:15:28,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/a61760338d9f410ba822bf82b84ed514 2024-12-08T00:15:28,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/8744dcb4e45c4b16a93d81bed9ff3805 is 50, key is test_row_0/B:col10/1733616927867/Put/seqid=0 2024-12-08T00:15:28,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742086_1262 (size=12301) 2024-12-08T00:15:28,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/8744dcb4e45c4b16a93d81bed9ff3805 2024-12-08T00:15:28,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/cbc9740cb79547ae8f6126519e311fc0 is 50, key is test_row_0/C:col10/1733616927867/Put/seqid=0 2024-12-08T00:15:28,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616988321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616988321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616988321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616988331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616988331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742087_1263 (size=12301) 2024-12-08T00:15:28,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616988535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616988535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616988535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616988541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616988541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/cbc9740cb79547ae8f6126519e311fc0 2024-12-08T00:15:28,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/a61760338d9f410ba822bf82b84ed514 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a61760338d9f410ba822bf82b84ed514 2024-12-08T00:15:28,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a61760338d9f410ba822bf82b84ed514, entries=150, sequenceid=333, filesize=12.0 K 2024-12-08T00:15:28,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/8744dcb4e45c4b16a93d81bed9ff3805 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8744dcb4e45c4b16a93d81bed9ff3805 2024-12-08T00:15:28,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8744dcb4e45c4b16a93d81bed9ff3805, entries=150, sequenceid=333, filesize=12.0 K 2024-12-08T00:15:28,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/cbc9740cb79547ae8f6126519e311fc0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cbc9740cb79547ae8f6126519e311fc0 2024-12-08T00:15:28,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cbc9740cb79547ae8f6126519e311fc0, entries=150, sequenceid=333, filesize=12.0 K 2024-12-08T00:15:28,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 582ms, sequenceid=333, compaction requested=true 2024-12-08T00:15:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:28,776 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:28,776 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:28,777 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:28,777 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:28,778 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/A is initiating minor compaction (all files) 2024-12-08T00:15:28,778 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/B is initiating minor compaction (all files) 2024-12-08T00:15:28,778 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/A in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:28,778 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/B in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:28,778 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c2789acb41df459e91769c7f890c116d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/4a3495f933ff4e14b0af5b7d98442f1f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/73563e83dd9e46b4a841295ee23bfeb8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8744dcb4e45c4b16a93d81bed9ff3805] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=48.5 K 2024-12-08T00:15:28,778 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/3de174aab21f451e9eafbbf16ce2e573, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7f76ff5e8e7b4d17a393e797da297c4c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7269e306efe14e1c938a8093473454f5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a61760338d9f410ba822bf82b84ed514] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=48.5 K 2024-12-08T00:15:28,779 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de174aab21f451e9eafbbf16ce2e573, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733616923848 2024-12-08T00:15:28,779 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c2789acb41df459e91769c7f890c116d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733616923848 2024-12-08T00:15:28,779 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f76ff5e8e7b4d17a393e797da297c4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733616925009 2024-12-08T00:15:28,779 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3495f933ff4e14b0af5b7d98442f1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733616925009 2024-12-08T00:15:28,779 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7269e306efe14e1c938a8093473454f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733616925655 2024-12-08T00:15:28,779 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 73563e83dd9e46b4a841295ee23bfeb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733616925655 2024-12-08T00:15:28,780 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting a61760338d9f410ba822bf82b84ed514, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733616927867 2024-12-08T00:15:28,780 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8744dcb4e45c4b16a93d81bed9ff3805, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733616927867 2024-12-08T00:15:28,797 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:28,798 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/8d4499b8fb904b6cbceb791938fa01e5 is 50, key is test_row_0/B:col10/1733616927867/Put/seqid=0 2024-12-08T00:15:28,799 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:28,799 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/b06ce9daedf24283934c68cd4b403d63 is 50, key is test_row_0/A:col10/1733616927867/Put/seqid=0 2024-12-08T00:15:28,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742088_1264 (size=12915) 2024-12-08T00:15:28,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742089_1265 (size=12915) 2024-12-08T00:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:28,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T00:15:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:28,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/94ea215094aa4e0088aae30b2378c8be is 50, key is test_row_0/A:col10/1733616928843/Put/seqid=0 2024-12-08T00:15:28,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616988866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616988868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616988869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616988871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616988871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742090_1266 (size=14741) 2024-12-08T00:15:28,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/94ea215094aa4e0088aae30b2378c8be 2024-12-08T00:15:28,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/56898044f3814384b5d93e03d08f44f1 is 50, key is test_row_0/B:col10/1733616928843/Put/seqid=0 2024-12-08T00:15:28,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742091_1267 (size=12301) 2024-12-08T00:15:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T00:15:28,921 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:28,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-08T00:15:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T00:15:28,924 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:28,924 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:28,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:28,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616988982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616988982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616988982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616988982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616988982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T00:15:29,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T00:15:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:29,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616989190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616989190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616989191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616989191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616989191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,224 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/8d4499b8fb904b6cbceb791938fa01e5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8d4499b8fb904b6cbceb791938fa01e5 2024-12-08T00:15:29,228 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/b06ce9daedf24283934c68cd4b403d63 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b06ce9daedf24283934c68cd4b403d63 2024-12-08T00:15:29,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T00:15:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,231 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/B of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 8d4499b8fb904b6cbceb791938fa01e5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:29,231 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,231 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/B, priority=12, startTime=1733616928776; duration=0sec 2024-12-08T00:15:29,231 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:29,231 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:B 2024-12-08T00:15:29,231 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:29,234 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/A of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into b06ce9daedf24283934c68cd4b403d63(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:29,234 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:29,235 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/A, priority=12, startTime=1733616928776; duration=0sec 2024-12-08T00:15:29,235 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:29,235 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:A 2024-12-08T00:15:29,235 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:29,235 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/C is initiating minor compaction (all files) 2024-12-08T00:15:29,235 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/C in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,235 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/45a39b5298054a82b688ed717719d342, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/69901d4ae3844bc0a82322fcfadc53cd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/115cceaaf4114ef785378c8d15737d1d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cbc9740cb79547ae8f6126519e311fc0] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=48.5 K 2024-12-08T00:15:29,236 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 45a39b5298054a82b688ed717719d342, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733616923848 2024-12-08T00:15:29,236 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 69901d4ae3844bc0a82322fcfadc53cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733616925009 2024-12-08T00:15:29,236 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 115cceaaf4114ef785378c8d15737d1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733616925655 2024-12-08T00:15:29,236 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc9740cb79547ae8f6126519e311fc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733616927867 2024-12-08T00:15:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T00:15:29,242 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:29,243 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/407e3ddfc50c49ad98583474a37bfff8 is 50, key is test_row_0/C:col10/1733616927867/Put/seqid=0 2024-12-08T00:15:29,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742092_1268 (size=12915) 2024-12-08T00:15:29,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/56898044f3814384b5d93e03d08f44f1 2024-12-08T00:15:29,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a3ba810b0a2a4fba82bb07e5a78f7466 is 50, key is test_row_0/C:col10/1733616928843/Put/seqid=0 2024-12-08T00:15:29,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742093_1269 (size=12301) 2024-12-08T00:15:29,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T00:15:29,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:29,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616989501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616989501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616989501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616989501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616989501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:29,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T00:15:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T00:15:29,651 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/407e3ddfc50c49ad98583474a37bfff8 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/407e3ddfc50c49ad98583474a37bfff8 2024-12-08T00:15:29,655 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/C of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 407e3ddfc50c49ad98583474a37bfff8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:29,655 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:29,655 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/C, priority=12, startTime=1733616928776; duration=0sec 2024-12-08T00:15:29,655 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:29,655 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:C 2024-12-08T00:15:29,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T00:15:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:29,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a3ba810b0a2a4fba82bb07e5a78f7466 2024-12-08T00:15:29,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/94ea215094aa4e0088aae30b2378c8be as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/94ea215094aa4e0088aae30b2378c8be 2024-12-08T00:15:29,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/94ea215094aa4e0088aae30b2378c8be, entries=200, sequenceid=345, filesize=14.4 K 2024-12-08T00:15:29,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/56898044f3814384b5d93e03d08f44f1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/56898044f3814384b5d93e03d08f44f1 2024-12-08T00:15:29,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/56898044f3814384b5d93e03d08f44f1, entries=150, sequenceid=345, filesize=12.0 K 2024-12-08T00:15:29,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a3ba810b0a2a4fba82bb07e5a78f7466 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a3ba810b0a2a4fba82bb07e5a78f7466 2024-12-08T00:15:29,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a3ba810b0a2a4fba82bb07e5a78f7466, entries=150, sequenceid=345, filesize=12.0 K 2024-12-08T00:15:29,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 885ms, sequenceid=345, compaction requested=false 2024-12-08T00:15:29,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:29,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T00:15:29,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:29,840 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/12b7911c8ad14d27aaaef90f0f305fec is 50, key is test_row_0/A:col10/1733616928869/Put/seqid=0 2024-12-08T00:15:29,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742094_1270 (size=12301) 2024-12-08T00:15:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:30,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:30,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616990019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616990019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616990021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616990020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616990021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T00:15:30,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616990130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616990130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616990130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616990131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616990131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,249 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/12b7911c8ad14d27aaaef90f0f305fec 2024-12-08T00:15:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/c46e05034dd84903a77c5c6be3f41066 is 50, key is test_row_0/B:col10/1733616928869/Put/seqid=0 2024-12-08T00:15:30,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742095_1271 (size=12301) 2024-12-08T00:15:30,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616990340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616990340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616990340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616990341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616990341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616990651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616990651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616990651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616990651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616990651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:30,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/c46e05034dd84903a77c5c6be3f41066 2024-12-08T00:15:30,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/b6caf5136ad5404fa89b87fc6c1e1078 is 50, key is test_row_0/C:col10/1733616928869/Put/seqid=0 2024-12-08T00:15:30,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742096_1272 (size=12301) 2024-12-08T00:15:31,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/b6caf5136ad5404fa89b87fc6c1e1078 2024-12-08T00:15:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T00:15:31,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/12b7911c8ad14d27aaaef90f0f305fec as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/12b7911c8ad14d27aaaef90f0f305fec 2024-12-08T00:15:31,094 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/12b7911c8ad14d27aaaef90f0f305fec, entries=150, sequenceid=368, filesize=12.0 K 2024-12-08T00:15:31,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/c46e05034dd84903a77c5c6be3f41066 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c46e05034dd84903a77c5c6be3f41066 2024-12-08T00:15:31,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c46e05034dd84903a77c5c6be3f41066, entries=150, sequenceid=368, filesize=12.0 K 2024-12-08T00:15:31,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/b6caf5136ad5404fa89b87fc6c1e1078 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b6caf5136ad5404fa89b87fc6c1e1078 2024-12-08T00:15:31,108 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b6caf5136ad5404fa89b87fc6c1e1078, entries=150, sequenceid=368, filesize=12.0 K 2024-12-08T00:15:31,109 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1268ms, sequenceid=368, compaction requested=true 2024-12-08T00:15:31,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:31,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:31,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-08T00:15:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-08T00:15:31,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-08T00:15:31,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1860 sec 2024-12-08T00:15:31,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1920 sec 2024-12-08T00:15:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:31,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T00:15:31,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:31,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:31,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:31,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:31,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:31,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:31,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/e27ce0bd39de496f9c2794ce37137f60 is 50, key is test_row_0/A:col10/1733616930020/Put/seqid=0 2024-12-08T00:15:31,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742097_1273 (size=12301) 2024-12-08T00:15:31,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616991179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616991180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616991184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616991184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616991185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616991290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616991291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616991291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616991291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616991301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616991500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616991500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616991501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616991501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616991511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/e27ce0bd39de496f9c2794ce37137f60 2024-12-08T00:15:31,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/e6ea611ba2dd4d9ea0c606f74b747b16 is 50, key is test_row_0/B:col10/1733616930020/Put/seqid=0 2024-12-08T00:15:31,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742098_1274 (size=12301) 2024-12-08T00:15:31,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616991811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616991811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616991811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616991812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:31,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616991821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:31,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:31,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:32,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/e6ea611ba2dd4d9ea0c606f74b747b16 2024-12-08T00:15:32,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/c42d9ee8c3c04e64a72b74817939c08a is 50, key is test_row_0/C:col10/1733616930020/Put/seqid=0 2024-12-08T00:15:32,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742099_1275 (size=12301) 2024-12-08T00:15:32,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:32,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616992321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616992320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:32,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616992321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:32,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616992321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:32,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616992331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:32,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/c42d9ee8c3c04e64a72b74817939c08a 2024-12-08T00:15:32,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/e27ce0bd39de496f9c2794ce37137f60 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/e27ce0bd39de496f9c2794ce37137f60 2024-12-08T00:15:32,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/e27ce0bd39de496f9c2794ce37137f60, entries=150, sequenceid=382, filesize=12.0 K 2024-12-08T00:15:32,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/e6ea611ba2dd4d9ea0c606f74b747b16 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e6ea611ba2dd4d9ea0c606f74b747b16 2024-12-08T00:15:32,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e6ea611ba2dd4d9ea0c606f74b747b16, entries=150, sequenceid=382, filesize=12.0 K 2024-12-08T00:15:32,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/c42d9ee8c3c04e64a72b74817939c08a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/c42d9ee8c3c04e64a72b74817939c08a 2024-12-08T00:15:32,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/c42d9ee8c3c04e64a72b74817939c08a, entries=150, sequenceid=382, filesize=12.0 K 2024-12-08T00:15:32,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1286ms, sequenceid=382, compaction requested=true 2024-12-08T00:15:32,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:32,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:32,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:32,449 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:32,449 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:32,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:32,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:32,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:32,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:32,451 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:32,451 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/A is initiating minor compaction (all files) 2024-12-08T00:15:32,451 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/A in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:32,451 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b06ce9daedf24283934c68cd4b403d63, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/94ea215094aa4e0088aae30b2378c8be, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/12b7911c8ad14d27aaaef90f0f305fec, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/e27ce0bd39de496f9c2794ce37137f60] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=51.0 K 2024-12-08T00:15:32,451 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:32,451 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting b06ce9daedf24283934c68cd4b403d63, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733616927867 2024-12-08T00:15:32,451 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/B is initiating minor compaction (all files) 2024-12-08T00:15:32,451 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/B in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:32,451 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8d4499b8fb904b6cbceb791938fa01e5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/56898044f3814384b5d93e03d08f44f1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c46e05034dd84903a77c5c6be3f41066, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e6ea611ba2dd4d9ea0c606f74b747b16] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=48.7 K 2024-12-08T00:15:32,452 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94ea215094aa4e0088aae30b2378c8be, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733616928209 2024-12-08T00:15:32,452 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d4499b8fb904b6cbceb791938fa01e5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733616927867 2024-12-08T00:15:32,452 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 56898044f3814384b5d93e03d08f44f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733616928211 2024-12-08T00:15:32,452 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12b7911c8ad14d27aaaef90f0f305fec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733616928864 2024-12-08T00:15:32,452 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c46e05034dd84903a77c5c6be3f41066, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733616928864 2024-12-08T00:15:32,452 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting e27ce0bd39de496f9c2794ce37137f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733616930019 2024-12-08T00:15:32,452 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ea611ba2dd4d9ea0c606f74b747b16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733616930019 2024-12-08T00:15:32,463 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:32,463 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/f0cb031349f74d5e80017fadfebc9e6f is 50, key is test_row_0/B:col10/1733616930020/Put/seqid=0 2024-12-08T00:15:32,470 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:32,471 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/cd2326c644fc4c09b4060f0822de00b6 is 50, key is test_row_0/A:col10/1733616930020/Put/seqid=0 2024-12-08T00:15:32,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742100_1276 (size=13051) 2024-12-08T00:15:32,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742101_1277 (size=13051) 2024-12-08T00:15:32,882 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/f0cb031349f74d5e80017fadfebc9e6f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f0cb031349f74d5e80017fadfebc9e6f 2024-12-08T00:15:32,886 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/B of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into f0cb031349f74d5e80017fadfebc9e6f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:32,886 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:32,886 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/B, priority=12, startTime=1733616932449; duration=0sec 2024-12-08T00:15:32,886 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:32,886 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:B 2024-12-08T00:15:32,887 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:15:32,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:15:32,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7cf59c9ae144cfe9fe4bafb7bc62bf46/C is initiating minor compaction (all files) 2024-12-08T00:15:32,888 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cf59c9ae144cfe9fe4bafb7bc62bf46/C in TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:32,888 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/407e3ddfc50c49ad98583474a37bfff8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a3ba810b0a2a4fba82bb07e5a78f7466, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b6caf5136ad5404fa89b87fc6c1e1078, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/c42d9ee8c3c04e64a72b74817939c08a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp, totalSize=48.7 K 2024-12-08T00:15:32,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 407e3ddfc50c49ad98583474a37bfff8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733616927867 2024-12-08T00:15:32,889 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a3ba810b0a2a4fba82bb07e5a78f7466, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733616928211 2024-12-08T00:15:32,889 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting b6caf5136ad5404fa89b87fc6c1e1078, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733616928864 2024-12-08T00:15:32,889 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c42d9ee8c3c04e64a72b74817939c08a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733616930019 2024-12-08T00:15:32,890 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/cd2326c644fc4c09b4060f0822de00b6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/cd2326c644fc4c09b4060f0822de00b6 2024-12-08T00:15:32,894 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/A of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into cd2326c644fc4c09b4060f0822de00b6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:32,894 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:32,894 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/A, priority=12, startTime=1733616932449; duration=0sec 2024-12-08T00:15:32,894 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:32,894 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:A 2024-12-08T00:15:32,897 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cf59c9ae144cfe9fe4bafb7bc62bf46#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:32,898 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/2f94d38aff074ef8b45c7b9adc54e210 is 50, key is test_row_0/C:col10/1733616930020/Put/seqid=0 2024-12-08T00:15:32,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742102_1278 (size=13051) 2024-12-08T00:15:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T00:15:33,091 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:33,091 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-08T00:15:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T00:15:33,092 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:33,093 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:33,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T00:15:33,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T00:15:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:33,246 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T00:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/ddfaa48d084a4835880e64f20d8cf0ac is 50, key is test_row_0/A:col10/1733616931182/Put/seqid=0 2024-12-08T00:15:33,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742103_1279 (size=12301) 2024-12-08T00:15:33,306 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/2f94d38aff074ef8b45c7b9adc54e210 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/2f94d38aff074ef8b45c7b9adc54e210 2024-12-08T00:15:33,310 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cf59c9ae144cfe9fe4bafb7bc62bf46/C of 7cf59c9ae144cfe9fe4bafb7bc62bf46 into 2f94d38aff074ef8b45c7b9adc54e210(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:33,310 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:33,310 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., storeName=7cf59c9ae144cfe9fe4bafb7bc62bf46/C, priority=12, startTime=1733616932450; duration=0sec 2024-12-08T00:15:33,310 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:33,310 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:C 2024-12-08T00:15:33,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:33,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. as already flushing 2024-12-08T00:15:33,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616993350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616993350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616993350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46540 deadline: 1733616993351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616993351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,355 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 7cf59c9ae144cfe9fe4bafb7bc62bf46 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T00:15:33,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616993461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616993461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616993461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616993461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,656 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/ddfaa48d084a4835880e64f20d8cf0ac 2024-12-08T00:15:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/e0f58d22e6d145e68d9fc77e96294de7 is 50, key is test_row_0/B:col10/1733616931182/Put/seqid=0 2024-12-08T00:15:33,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742104_1280 (size=12301) 2024-12-08T00:15:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616993670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616993670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616993671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616993671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T00:15:33,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616993981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616993981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616993981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616993981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:34,066 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/e0f58d22e6d145e68d9fc77e96294de7 2024-12-08T00:15:34,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/0a9ce83abcca4a91aa2fe95da351b259 is 50, key is test_row_0/C:col10/1733616931182/Put/seqid=0 2024-12-08T00:15:34,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742105_1281 (size=12301) 2024-12-08T00:15:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T00:15:34,476 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/0a9ce83abcca4a91aa2fe95da351b259 2024-12-08T00:15:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/ddfaa48d084a4835880e64f20d8cf0ac as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ddfaa48d084a4835880e64f20d8cf0ac 2024-12-08T00:15:34,484 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ddfaa48d084a4835880e64f20d8cf0ac, entries=150, sequenceid=402, filesize=12.0 K 2024-12-08T00:15:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/e0f58d22e6d145e68d9fc77e96294de7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e0f58d22e6d145e68d9fc77e96294de7 2024-12-08T00:15:34,489 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e0f58d22e6d145e68d9fc77e96294de7, entries=150, sequenceid=402, filesize=12.0 K 2024-12-08T00:15:34,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/0a9ce83abcca4a91aa2fe95da351b259 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/0a9ce83abcca4a91aa2fe95da351b259 2024-12-08T00:15:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46538 deadline: 1733616994491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1733616994492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:34,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1733616994492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:34,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46566 deadline: 1733616994492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:34,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:34,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:34,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:15:34,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/0a9ce83abcca4a91aa2fe95da351b259, entries=150, sequenceid=402, filesize=12.0 K 2024-12-08T00:15:34,496 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1250ms, sequenceid=402, compaction requested=false 2024-12-08T00:15:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:34,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:34,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-08T00:15:34,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-08T00:15:34,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-08T00:15:34,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-12-08T00:15:34,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4090 sec 2024-12-08T00:15:35,048 INFO [Thread-1006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1006. 2024-12-08T00:15:35,048 DEBUG [Thread-1006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,048 DEBUG [Thread-1006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,048 INFO [Thread-1012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1012. 2024-12-08T00:15:35,048 DEBUG [Thread-1012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,048 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,048 INFO [Thread-1004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1004. 2024-12-08T00:15:35,048 DEBUG [Thread-1004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,049 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,049 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,049 DEBUG [Thread-1006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,049 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,049 INFO [Thread-1010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1010. 2024-12-08T00:15:35,049 DEBUG [Thread-1010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,049 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,050 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,050 INFO [Thread-1008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1008. 2024-12-08T00:15:35,050 DEBUG [Thread-1008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,050 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,050 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T00:15:35,241 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:35,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T00:15:35,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:35,373 INFO [Thread-995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-995. 2024-12-08T00:15:35,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:35,373 DEBUG [Thread-995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,373 DEBUG [Thread-995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:35,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:35,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:35,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:35,373 DEBUG [Thread-995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/d7d44b2de8444ea28358dd8ce50d6271 is 50, key is test_row_0/A:col10/1733616933348/Put/seqid=0 2024-12-08T00:15:35,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742106_1282 (size=12301) 2024-12-08T00:15:35,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/d7d44b2de8444ea28358dd8ce50d6271 2024-12-08T00:15:35,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/030ac64b86fc4232bb43e3f274c9b3b2 is 50, key is test_row_0/B:col10/1733616933348/Put/seqid=0 2024-12-08T00:15:35,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742107_1283 (size=12301) 2024-12-08T00:15:35,503 INFO [Thread-1001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1001. 2024-12-08T00:15:35,503 INFO [Thread-997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-997. 2024-12-08T00:15:35,503 DEBUG [Thread-997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,503 DEBUG [Thread-1001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,503 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,503 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,503 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,503 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,515 INFO [Thread-999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-999. 2024-12-08T00:15:35,515 INFO [Thread-993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-993. 2024-12-08T00:15:35,515 DEBUG [Thread-999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,515 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,515 DEBUG [Thread-993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:15:35,515 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,515 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,515 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4008 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3928 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3835 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4015 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3892 2024-12-08T00:15:35,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T00:15:35,516 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T00:15:35,516 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T00:15:35,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:35,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:15:35,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T00:15:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T00:15:35,521 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616935521"}]},"ts":"1733616935521"} 2024-12-08T00:15:35,522 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T00:15:35,522 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T00:15:35,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T00:15:35,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cf59c9ae144cfe9fe4bafb7bc62bf46, UNASSIGN}] 2024-12-08T00:15:35,529 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cf59c9ae144cfe9fe4bafb7bc62bf46, UNASSIGN 2024-12-08T00:15:35,530 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=7cf59c9ae144cfe9fe4bafb7bc62bf46, regionState=CLOSING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:35,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cf59c9ae144cfe9fe4bafb7bc62bf46, UNASSIGN because future has completed 2024-12-08T00:15:35,532 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T00:15:35,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:15:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T00:15:35,686 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:35,686 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T00:15:35,686 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 7cf59c9ae144cfe9fe4bafb7bc62bf46, disabling compactions & flushes 2024-12-08T00:15:35,686 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:35,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/030ac64b86fc4232bb43e3f274c9b3b2 2024-12-08T00:15:35,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/5de5750ac55442e09a0c6429fccf69b0 is 50, key is test_row_0/C:col10/1733616933348/Put/seqid=0 2024-12-08T00:15:35,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742108_1284 (size=12301) 2024-12-08T00:15:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T00:15:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T00:15:36,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/5de5750ac55442e09a0c6429fccf69b0 2024-12-08T00:15:36,243 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T00:15:36,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/d7d44b2de8444ea28358dd8ce50d6271 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/d7d44b2de8444ea28358dd8ce50d6271 2024-12-08T00:15:36,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/d7d44b2de8444ea28358dd8ce50d6271, entries=150, sequenceid=418, filesize=12.0 K 2024-12-08T00:15:36,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/030ac64b86fc4232bb43e3f274c9b3b2 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/030ac64b86fc4232bb43e3f274c9b3b2 2024-12-08T00:15:36,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/030ac64b86fc4232bb43e3f274c9b3b2, entries=150, sequenceid=418, filesize=12.0 K 2024-12-08T00:15:36,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/5de5750ac55442e09a0c6429fccf69b0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5de5750ac55442e09a0c6429fccf69b0 2024-12-08T00:15:36,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5de5750ac55442e09a0c6429fccf69b0, entries=150, sequenceid=418, filesize=12.0 K 2024-12-08T00:15:36,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 893ms, sequenceid=418, compaction requested=true 2024-12-08T00:15:36,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: 2024-12-08T00:15:36,267 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:36,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:36,267 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:36,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:36,267 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. because compaction request was cancelled 2024-12-08T00:15:36,267 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. after waiting 0 ms 2024-12-08T00:15:36,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:36,267 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:A 2024-12-08T00:15:36,267 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:36,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:36,267 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. because compaction request was cancelled 2024-12-08T00:15:36,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cf59c9ae144cfe9fe4bafb7bc62bf46:C, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:36,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:36,267 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:B 2024-12-08T00:15:36,267 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. because compaction request was cancelled 2024-12-08T00:15:36,267 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cf59c9ae144cfe9fe4bafb7bc62bf46:C 2024-12-08T00:15:36,267 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-08T00:15:36,267 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=A 2024-12-08T00:15:36,267 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:36,267 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=B 2024-12-08T00:15:36,267 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:36,267 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cf59c9ae144cfe9fe4bafb7bc62bf46, store=C 2024-12-08T00:15:36,267 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:36,270 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/881b5d493ef24fc89e3a20d193363600 is 50, key is test_row_1/A:col10/1733616935512/Put/seqid=0 2024-12-08T00:15:36,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742109_1285 (size=9857) 2024-12-08T00:15:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T00:15:36,685 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/881b5d493ef24fc89e3a20d193363600 2024-12-08T00:15:36,697 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/ef62067196b84b42acbd6adc95b325ec is 50, key is test_row_1/B:col10/1733616935512/Put/seqid=0 2024-12-08T00:15:36,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742110_1286 (size=9857) 2024-12-08T00:15:37,101 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/ef62067196b84b42acbd6adc95b325ec 2024-12-08T00:15:37,108 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a52c1df2673244ac84d2db6d4e9d5939 is 50, key is test_row_1/C:col10/1733616935512/Put/seqid=0 2024-12-08T00:15:37,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742111_1287 (size=9857) 2024-12-08T00:15:37,512 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a52c1df2673244ac84d2db6d4e9d5939 2024-12-08T00:15:37,519 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/A/881b5d493ef24fc89e3a20d193363600 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/881b5d493ef24fc89e3a20d193363600 2024-12-08T00:15:37,527 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/881b5d493ef24fc89e3a20d193363600, entries=100, sequenceid=425, filesize=9.6 K 2024-12-08T00:15:37,528 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/B/ef62067196b84b42acbd6adc95b325ec as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/ef62067196b84b42acbd6adc95b325ec 2024-12-08T00:15:37,533 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/ef62067196b84b42acbd6adc95b325ec, entries=100, sequenceid=425, filesize=9.6 K 2024-12-08T00:15:37,535 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/.tmp/C/a52c1df2673244ac84d2db6d4e9d5939 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a52c1df2673244ac84d2db6d4e9d5939 2024-12-08T00:15:37,541 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a52c1df2673244ac84d2db6d4e9d5939, entries=100, sequenceid=425, filesize=9.6 K 2024-12-08T00:15:37,541 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1274ms, sequenceid=425, compaction requested=true 2024-12-08T00:15:37,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/de4b43b6e6154084867321603f3f9a8e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/075f793d50474882974e52972a636189, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/2eadcf078fad43298efa5ecc6299046d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1b72bbcbdcf54195b85b3fc7567e8238, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/83aa11a7ce21408ca96eddbda6d0cad8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a60b5127a4414cc4ad7dc3a05063297a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/c3cf570af7a3483ab82327114f84eaf9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b7654030927d4e8cb387c84fccb5c61f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ab2363a22a414103953ba32f4531d0f0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1e53d20f0200457b83200637461bde07, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/46e2cd5ed52347fcb0b107465f019913, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/57be3ef3e5e44b1e87f37ba0fd705b32, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a8ee7e4512aa4f83a0b0c6c6355edf9c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/3de174aab21f451e9eafbbf16ce2e573, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/683940345d0b4e8492f45a773f9ff6b9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7f76ff5e8e7b4d17a393e797da297c4c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7269e306efe14e1c938a8093473454f5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b06ce9daedf24283934c68cd4b403d63, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a61760338d9f410ba822bf82b84ed514, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/94ea215094aa4e0088aae30b2378c8be, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/12b7911c8ad14d27aaaef90f0f305fec, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/e27ce0bd39de496f9c2794ce37137f60] to archive 2024-12-08T00:15:37,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:15:37,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/de4b43b6e6154084867321603f3f9a8e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/de4b43b6e6154084867321603f3f9a8e 2024-12-08T00:15:37,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/075f793d50474882974e52972a636189 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/075f793d50474882974e52972a636189 2024-12-08T00:15:37,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/2eadcf078fad43298efa5ecc6299046d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/2eadcf078fad43298efa5ecc6299046d 2024-12-08T00:15:37,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1b72bbcbdcf54195b85b3fc7567e8238 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1b72bbcbdcf54195b85b3fc7567e8238 2024-12-08T00:15:37,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/83aa11a7ce21408ca96eddbda6d0cad8 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/83aa11a7ce21408ca96eddbda6d0cad8 2024-12-08T00:15:37,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a60b5127a4414cc4ad7dc3a05063297a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a60b5127a4414cc4ad7dc3a05063297a 2024-12-08T00:15:37,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/c3cf570af7a3483ab82327114f84eaf9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/c3cf570af7a3483ab82327114f84eaf9 2024-12-08T00:15:37,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b7654030927d4e8cb387c84fccb5c61f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b7654030927d4e8cb387c84fccb5c61f 2024-12-08T00:15:37,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ab2363a22a414103953ba32f4531d0f0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ab2363a22a414103953ba32f4531d0f0 2024-12-08T00:15:37,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1e53d20f0200457b83200637461bde07 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/1e53d20f0200457b83200637461bde07 2024-12-08T00:15:37,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/46e2cd5ed52347fcb0b107465f019913 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/46e2cd5ed52347fcb0b107465f019913 2024-12-08T00:15:37,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/57be3ef3e5e44b1e87f37ba0fd705b32 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/57be3ef3e5e44b1e87f37ba0fd705b32 2024-12-08T00:15:37,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a8ee7e4512aa4f83a0b0c6c6355edf9c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a8ee7e4512aa4f83a0b0c6c6355edf9c 2024-12-08T00:15:37,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/3de174aab21f451e9eafbbf16ce2e573 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/3de174aab21f451e9eafbbf16ce2e573 2024-12-08T00:15:37,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/683940345d0b4e8492f45a773f9ff6b9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/683940345d0b4e8492f45a773f9ff6b9 2024-12-08T00:15:37,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7f76ff5e8e7b4d17a393e797da297c4c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7f76ff5e8e7b4d17a393e797da297c4c 2024-12-08T00:15:37,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7269e306efe14e1c938a8093473454f5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/7269e306efe14e1c938a8093473454f5 2024-12-08T00:15:37,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b06ce9daedf24283934c68cd4b403d63 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/b06ce9daedf24283934c68cd4b403d63 2024-12-08T00:15:37,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a61760338d9f410ba822bf82b84ed514 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/a61760338d9f410ba822bf82b84ed514 2024-12-08T00:15:37,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/94ea215094aa4e0088aae30b2378c8be to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/94ea215094aa4e0088aae30b2378c8be 2024-12-08T00:15:37,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/12b7911c8ad14d27aaaef90f0f305fec to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/12b7911c8ad14d27aaaef90f0f305fec 2024-12-08T00:15:37,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/e27ce0bd39de496f9c2794ce37137f60 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/e27ce0bd39de496f9c2794ce37137f60 2024-12-08T00:15:37,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb773222ed84b26a67508e465358607, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e90edd2f58c45c29c5d7cacce5f46b3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/2d163be4416e45278db9959a1a910b17, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/d1c6765ebf8b46e2ae04a50d906569bf, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e80055cbc1a4e798ab898501e98618d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/bbbc16f8b4014f0fb40428d005f6db5d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f45df10247cf4803bf045419fbc01449, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/430b070feb194930adec7b1ce080ebac, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/47b419653e6d47a2b4ea487241951e52, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02c990626d1c4c919a32001efcc3cde7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02a3dc647070416eacfd06c4a175749b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb6a36deba94b1d94d49292797b09f3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8c594e26c08144cb8d7905e74bfd1864, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c2789acb41df459e91769c7f890c116d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f5c93e7b19364875934bda610f54d254, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/4a3495f933ff4e14b0af5b7d98442f1f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/73563e83dd9e46b4a841295ee23bfeb8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8d4499b8fb904b6cbceb791938fa01e5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8744dcb4e45c4b16a93d81bed9ff3805, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/56898044f3814384b5d93e03d08f44f1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c46e05034dd84903a77c5c6be3f41066, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e6ea611ba2dd4d9ea0c606f74b747b16] to archive 2024-12-08T00:15:37,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:15:37,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb773222ed84b26a67508e465358607 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb773222ed84b26a67508e465358607 2024-12-08T00:15:37,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e90edd2f58c45c29c5d7cacce5f46b3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e90edd2f58c45c29c5d7cacce5f46b3 2024-12-08T00:15:37,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/2d163be4416e45278db9959a1a910b17 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/2d163be4416e45278db9959a1a910b17 2024-12-08T00:15:37,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/d1c6765ebf8b46e2ae04a50d906569bf to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/d1c6765ebf8b46e2ae04a50d906569bf 2024-12-08T00:15:37,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e80055cbc1a4e798ab898501e98618d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/1e80055cbc1a4e798ab898501e98618d 2024-12-08T00:15:37,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/bbbc16f8b4014f0fb40428d005f6db5d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/bbbc16f8b4014f0fb40428d005f6db5d 2024-12-08T00:15:37,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f45df10247cf4803bf045419fbc01449 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f45df10247cf4803bf045419fbc01449 2024-12-08T00:15:37,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/430b070feb194930adec7b1ce080ebac to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/430b070feb194930adec7b1ce080ebac 2024-12-08T00:15:37,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/47b419653e6d47a2b4ea487241951e52 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/47b419653e6d47a2b4ea487241951e52 2024-12-08T00:15:37,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02c990626d1c4c919a32001efcc3cde7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02c990626d1c4c919a32001efcc3cde7 2024-12-08T00:15:37,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02a3dc647070416eacfd06c4a175749b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/02a3dc647070416eacfd06c4a175749b 2024-12-08T00:15:37,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb6a36deba94b1d94d49292797b09f3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/6eb6a36deba94b1d94d49292797b09f3 2024-12-08T00:15:37,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8c594e26c08144cb8d7905e74bfd1864 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8c594e26c08144cb8d7905e74bfd1864 2024-12-08T00:15:37,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c2789acb41df459e91769c7f890c116d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c2789acb41df459e91769c7f890c116d 2024-12-08T00:15:37,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f5c93e7b19364875934bda610f54d254 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f5c93e7b19364875934bda610f54d254 2024-12-08T00:15:37,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/4a3495f933ff4e14b0af5b7d98442f1f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/4a3495f933ff4e14b0af5b7d98442f1f 2024-12-08T00:15:37,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/73563e83dd9e46b4a841295ee23bfeb8 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/73563e83dd9e46b4a841295ee23bfeb8 2024-12-08T00:15:37,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8d4499b8fb904b6cbceb791938fa01e5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8d4499b8fb904b6cbceb791938fa01e5 2024-12-08T00:15:37,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8744dcb4e45c4b16a93d81bed9ff3805 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/8744dcb4e45c4b16a93d81bed9ff3805 2024-12-08T00:15:37,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/56898044f3814384b5d93e03d08f44f1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/56898044f3814384b5d93e03d08f44f1 2024-12-08T00:15:37,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c46e05034dd84903a77c5c6be3f41066 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/c46e05034dd84903a77c5c6be3f41066 2024-12-08T00:15:37,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e6ea611ba2dd4d9ea0c606f74b747b16 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e6ea611ba2dd4d9ea0c606f74b747b16 2024-12-08T00:15:37,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a61d78c0b43a4550acc5baaebbe3812a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5755ccd49f994531b95ee3480b493680, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e8086cf053b3426693514814c4e915d7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e83c70d0d608486c861e2cbf657335a1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a8c998cda3a147bdad6ca7cfc38d0fcd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1a2a60c3f30b41d49617f70308f8def3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1ce5a2da26b048c489bdbdde9413d4a0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/2dc6be93e2794fe9b00fdad32955af30, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/23c10d6086c144bdb3deb5125d78818e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cdd83ac848524b94bcd32a6c65d53ca9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8f02732460ac4f108475a0e4b8786a69, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8c86807b8bba4ef7a63600573ac060e7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/9e868d1fb1254962bb69be9c59ff8357, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/45a39b5298054a82b688ed717719d342, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b82df1a4602e44a0b2a70262d2535ef7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/69901d4ae3844bc0a82322fcfadc53cd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/115cceaaf4114ef785378c8d15737d1d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/407e3ddfc50c49ad98583474a37bfff8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cbc9740cb79547ae8f6126519e311fc0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a3ba810b0a2a4fba82bb07e5a78f7466, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b6caf5136ad5404fa89b87fc6c1e1078, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/c42d9ee8c3c04e64a72b74817939c08a] to archive 2024-12-08T00:15:37,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:15:37,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a61d78c0b43a4550acc5baaebbe3812a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a61d78c0b43a4550acc5baaebbe3812a 2024-12-08T00:15:37,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5755ccd49f994531b95ee3480b493680 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5755ccd49f994531b95ee3480b493680 2024-12-08T00:15:37,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e8086cf053b3426693514814c4e915d7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e8086cf053b3426693514814c4e915d7 2024-12-08T00:15:37,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e83c70d0d608486c861e2cbf657335a1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/e83c70d0d608486c861e2cbf657335a1 2024-12-08T00:15:37,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a8c998cda3a147bdad6ca7cfc38d0fcd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a8c998cda3a147bdad6ca7cfc38d0fcd 2024-12-08T00:15:37,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1a2a60c3f30b41d49617f70308f8def3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1a2a60c3f30b41d49617f70308f8def3 2024-12-08T00:15:37,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1ce5a2da26b048c489bdbdde9413d4a0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/1ce5a2da26b048c489bdbdde9413d4a0 2024-12-08T00:15:37,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/2dc6be93e2794fe9b00fdad32955af30 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/2dc6be93e2794fe9b00fdad32955af30 2024-12-08T00:15:37,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/23c10d6086c144bdb3deb5125d78818e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/23c10d6086c144bdb3deb5125d78818e 2024-12-08T00:15:37,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cdd83ac848524b94bcd32a6c65d53ca9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cdd83ac848524b94bcd32a6c65d53ca9 2024-12-08T00:15:37,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8f02732460ac4f108475a0e4b8786a69 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8f02732460ac4f108475a0e4b8786a69 2024-12-08T00:15:37,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8c86807b8bba4ef7a63600573ac060e7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/8c86807b8bba4ef7a63600573ac060e7 2024-12-08T00:15:37,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/9e868d1fb1254962bb69be9c59ff8357 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/9e868d1fb1254962bb69be9c59ff8357 2024-12-08T00:15:37,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/45a39b5298054a82b688ed717719d342 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/45a39b5298054a82b688ed717719d342 2024-12-08T00:15:37,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b82df1a4602e44a0b2a70262d2535ef7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b82df1a4602e44a0b2a70262d2535ef7 2024-12-08T00:15:37,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/69901d4ae3844bc0a82322fcfadc53cd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/69901d4ae3844bc0a82322fcfadc53cd 2024-12-08T00:15:37,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/115cceaaf4114ef785378c8d15737d1d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/115cceaaf4114ef785378c8d15737d1d 2024-12-08T00:15:37,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/407e3ddfc50c49ad98583474a37bfff8 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/407e3ddfc50c49ad98583474a37bfff8 2024-12-08T00:15:37,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cbc9740cb79547ae8f6126519e311fc0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/cbc9740cb79547ae8f6126519e311fc0 2024-12-08T00:15:37,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a3ba810b0a2a4fba82bb07e5a78f7466 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a3ba810b0a2a4fba82bb07e5a78f7466 2024-12-08T00:15:37,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b6caf5136ad5404fa89b87fc6c1e1078 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/b6caf5136ad5404fa89b87fc6c1e1078 2024-12-08T00:15:37,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/c42d9ee8c3c04e64a72b74817939c08a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/c42d9ee8c3c04e64a72b74817939c08a 2024-12-08T00:15:37,646 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/recovered.edits/428.seqid, newMaxSeqId=428, maxSeqId=1 2024-12-08T00:15:37,647 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. 2024-12-08T00:15:37,647 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 7cf59c9ae144cfe9fe4bafb7bc62bf46: Waiting for close lock at 1733616935686Running coprocessor pre-close hooks at 1733616935686Disabling compacts and flushes for region at 1733616935686Disabling writes for close at 1733616936267 (+581 ms)Obtaining lock to block concurrent updates at 1733616936267Preparing flush snapshotting stores in 7cf59c9ae144cfe9fe4bafb7bc62bf46 at 1733616936267Finished memstore snapshotting TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733616936267Flushing stores of TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46. at 1733616936267Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46/A: creating writer at 1733616936267Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46/A: appending metadata at 1733616936270 (+3 ms)Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46/A: closing flushed file at 1733616936270Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46/B: creating writer at 1733616936694 (+424 ms)Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46/B: appending metadata at 1733616936697 (+3 ms)Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46/B: closing flushed file at 1733616936697Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46/C: creating writer at 1733616937105 (+408 ms)Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46/C: appending metadata at 1733616937108 (+3 ms)Flushing 7cf59c9ae144cfe9fe4bafb7bc62bf46/C: closing flushed file at 1733616937108Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ff5344: reopening flushed file at 1733616937518 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f8ade2b: reopening flushed file at 1733616937527 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ded76dc: reopening flushed file at 1733616937534 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7cf59c9ae144cfe9fe4bafb7bc62bf46 in 1274ms, sequenceid=425, compaction requested=true at 1733616937541 (+7 ms)Writing region close event to WAL at 1733616937643 (+102 ms)Running coprocessor post-close hooks at 1733616937646 (+3 ms)Closed at 1733616937647 (+1 ms) 2024-12-08T00:15:37,648 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:37,648 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=7cf59c9ae144cfe9fe4bafb7bc62bf46, regionState=CLOSED 2024-12-08T00:15:37,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:15:37,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-08T00:15:37,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 7cf59c9ae144cfe9fe4bafb7bc62bf46, server=c1a8cf8d4cfa,36021,1733616847970 in 2.1170 sec 2024-12-08T00:15:37,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-08T00:15:37,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cf59c9ae144cfe9fe4bafb7bc62bf46, UNASSIGN in 2.1260 sec 2024-12-08T00:15:37,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-08T00:15:37,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1300 sec 2024-12-08T00:15:37,656 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616937656"}]},"ts":"1733616937656"} 2024-12-08T00:15:37,658 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T00:15:37,658 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T00:15:37,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1410 sec 2024-12-08T00:15:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T00:15:37,681 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:37,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T00:15:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:37,683 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T00:15:37,684 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:37,687 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:37,690 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/recovered.edits] 2024-12-08T00:15:37,695 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/881b5d493ef24fc89e3a20d193363600 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/881b5d493ef24fc89e3a20d193363600 2024-12-08T00:15:37,697 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/cd2326c644fc4c09b4060f0822de00b6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/cd2326c644fc4c09b4060f0822de00b6 2024-12-08T00:15:37,699 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/d7d44b2de8444ea28358dd8ce50d6271 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/d7d44b2de8444ea28358dd8ce50d6271 2024-12-08T00:15:37,700 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ddfaa48d084a4835880e64f20d8cf0ac to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/A/ddfaa48d084a4835880e64f20d8cf0ac 2024-12-08T00:15:37,704 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/030ac64b86fc4232bb43e3f274c9b3b2 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/030ac64b86fc4232bb43e3f274c9b3b2 2024-12-08T00:15:37,705 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e0f58d22e6d145e68d9fc77e96294de7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/e0f58d22e6d145e68d9fc77e96294de7 2024-12-08T00:15:37,706 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/ef62067196b84b42acbd6adc95b325ec to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/ef62067196b84b42acbd6adc95b325ec 2024-12-08T00:15:37,706 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f0cb031349f74d5e80017fadfebc9e6f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/B/f0cb031349f74d5e80017fadfebc9e6f 2024-12-08T00:15:37,708 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/0a9ce83abcca4a91aa2fe95da351b259 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/0a9ce83abcca4a91aa2fe95da351b259 2024-12-08T00:15:37,709 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/2f94d38aff074ef8b45c7b9adc54e210 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/2f94d38aff074ef8b45c7b9adc54e210 2024-12-08T00:15:37,709 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5de5750ac55442e09a0c6429fccf69b0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/5de5750ac55442e09a0c6429fccf69b0 2024-12-08T00:15:37,710 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a52c1df2673244ac84d2db6d4e9d5939 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/C/a52c1df2673244ac84d2db6d4e9d5939 2024-12-08T00:15:37,712 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/recovered.edits/428.seqid to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46/recovered.edits/428.seqid 2024-12-08T00:15:37,713 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7cf59c9ae144cfe9fe4bafb7bc62bf46 2024-12-08T00:15:37,713 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T00:15:37,715 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:37,718 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T00:15:37,719 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T00:15:37,721 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:37,721 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T00:15:37,721 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733616937721"}]},"ts":"9223372036854775807"} 2024-12-08T00:15:37,723 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T00:15:37,723 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7cf59c9ae144cfe9fe4bafb7bc62bf46, NAME => 'TestAcidGuarantees,,1733616912814.7cf59c9ae144cfe9fe4bafb7bc62bf46.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T00:15:37,724 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T00:15:37,724 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733616937724"}]},"ts":"9223372036854775807"} 2024-12-08T00:15:37,726 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T00:15:37,727 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:37,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-12-08T00:15:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T00:15:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T00:15:37,792 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:37,803 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1076213060_22 at /127.0.0.1:44100 [Waiting for operation #294] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1975563135_22 at /127.0.0.1:57546 [Waiting for operation #200] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=462 (was 353) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8698 (was 9282) 2024-12-08T00:15:37,821 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=462, ProcessCount=11, AvailableMemoryMB=8697 2024-12-08T00:15:37,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T00:15:37,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:15:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:37,824 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T00:15:37,824 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:37,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-12-08T00:15:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-08T00:15:37,825 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T00:15:37,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742112_1288 (size=1061) 2024-12-08T00:15:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-08T00:15:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-08T00:15:38,232 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329 2024-12-08T00:15:38,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742113_1289 (size=53) 2024-12-08T00:15:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-08T00:15:38,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:15:38,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7404e7668f405a8889810821ba8518dd, disabling compactions & flushes 2024-12-08T00:15:38,638 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:38,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:38,639 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. after waiting 1 ms 2024-12-08T00:15:38,639 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:38,639 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:38,639 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7404e7668f405a8889810821ba8518dd: Waiting for close lock at 1733616938638Disabling compacts and flushes for region at 1733616938638Disabling writes for close at 1733616938639 (+1 ms)Writing region close event to WAL at 1733616938639Closed at 1733616938639 2024-12-08T00:15:38,643 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T00:15:38,643 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733616938643"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733616938643"}]},"ts":"1733616938643"} 2024-12-08T00:15:38,645 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T00:15:38,646 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T00:15:38,646 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616938646"}]},"ts":"1733616938646"} 2024-12-08T00:15:38,647 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T00:15:38,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, ASSIGN}] 2024-12-08T00:15:38,649 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, ASSIGN 2024-12-08T00:15:38,650 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, ASSIGN; state=OFFLINE, location=c1a8cf8d4cfa,36021,1733616847970; forceNewPlan=false, retain=false 2024-12-08T00:15:38,800 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=7404e7668f405a8889810821ba8518dd, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:38,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, ASSIGN because future has completed 2024-12-08T00:15:38,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:15:38,957 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:38,957 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:15:38,958 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,958 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:15:38,958 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,958 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,960 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,961 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:15:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-08T00:15:38,962 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7404e7668f405a8889810821ba8518dd columnFamilyName A 2024-12-08T00:15:38,962 DEBUG [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:38,963 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(327): Store=7404e7668f405a8889810821ba8518dd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:15:38,963 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,965 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:15:38,965 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7404e7668f405a8889810821ba8518dd columnFamilyName B 2024-12-08T00:15:38,965 DEBUG [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:38,966 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(327): Store=7404e7668f405a8889810821ba8518dd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:15:38,966 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,968 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:15:38,968 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7404e7668f405a8889810821ba8518dd columnFamilyName C 2024-12-08T00:15:38,968 DEBUG [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:38,969 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(327): Store=7404e7668f405a8889810821ba8518dd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:15:38,969 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:38,969 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,971 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,971 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,972 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,973 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,973 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:15:38,975 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,978 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T00:15:38,979 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 7404e7668f405a8889810821ba8518dd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59515694, jitterRate=-0.11314705014228821}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:15:38,979 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:38,980 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 7404e7668f405a8889810821ba8518dd: Running coprocessor pre-open hook at 1733616938958Writing region info on filesystem at 1733616938958Initializing all the Stores at 1733616938959 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616938959Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616938959Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616938959Cleaning up temporary data from old regions at 1733616938973 (+14 ms)Running coprocessor post-open hooks at 1733616938979 (+6 ms)Region opened successfully at 1733616938980 (+1 ms) 2024-12-08T00:15:38,981 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., pid=85, masterSystemTime=1733616938954 2024-12-08T00:15:38,985 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=7404e7668f405a8889810821ba8518dd, regionState=OPEN, openSeqNum=2, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:38,985 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:38,986 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:38,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:15:38,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-08T00:15:38,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 in 186 msec 2024-12-08T00:15:38,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-08T00:15:38,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, ASSIGN in 343 msec 2024-12-08T00:15:38,994 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T00:15:38,994 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616938994"}]},"ts":"1733616938994"} 2024-12-08T00:15:38,996 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T00:15:38,997 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T00:15:38,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-12-08T00:15:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-08T00:15:39,971 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:39,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:39,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:39,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:39,973 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:39,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:39,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:39,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d66eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:39,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:39,975 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f57163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:39,975 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:39,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:39,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:39,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:39,978 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T00:15:39,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:15:39,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@230dd7ac 2024-12-08T00:15:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T00:15:39,981 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T00:15:39,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T00:15:39,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:15:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T00:15:39,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742114_1290 (size=1097) 2024-12-08T00:15:40,399 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T00:15:40,399 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T00:15:40,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T00:15:40,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, REOPEN/MOVE}] 2024-12-08T00:15:40,408 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, REOPEN/MOVE 2024-12-08T00:15:40,409 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=7404e7668f405a8889810821ba8518dd, regionState=CLOSING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:40,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, REOPEN/MOVE because future has completed 2024-12-08T00:15:40,411 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T00:15:40,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:15:40,565 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,565 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T00:15:40,565 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 7404e7668f405a8889810821ba8518dd, disabling compactions & flushes 2024-12-08T00:15:40,565 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:40,565 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:40,565 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. after waiting 0 ms 2024-12-08T00:15:40,565 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:40,570 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T00:15:40,571 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:40,571 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 7404e7668f405a8889810821ba8518dd: Waiting for close lock at 1733616940565Running coprocessor pre-close hooks at 1733616940565Disabling compacts and flushes for region at 1733616940565Disabling writes for close at 1733616940565Writing region close event to WAL at 1733616940566 (+1 ms)Running coprocessor post-close hooks at 1733616940571 (+5 ms)Closed at 1733616940571 2024-12-08T00:15:40,571 WARN [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 7404e7668f405a8889810821ba8518dd to self. 2024-12-08T00:15:40,574 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,574 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=7404e7668f405a8889810821ba8518dd, regionState=CLOSED 2024-12-08T00:15:40,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:15:40,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-08T00:15:40,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 in 166 msec 2024-12-08T00:15:40,580 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, REOPEN/MOVE; state=CLOSED, location=c1a8cf8d4cfa,36021,1733616847970; forceNewPlan=false, retain=true 2024-12-08T00:15:40,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=7404e7668f405a8889810821ba8518dd, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:40,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, REOPEN/MOVE because future has completed 2024-12-08T00:15:40,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:15:40,896 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:40,896 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:15:40,896 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,897 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:15:40,897 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,897 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,899 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,900 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:15:40,900 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7404e7668f405a8889810821ba8518dd columnFamilyName A 2024-12-08T00:15:40,901 DEBUG [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:40,902 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(327): Store=7404e7668f405a8889810821ba8518dd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:15:40,902 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,903 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:15:40,903 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7404e7668f405a8889810821ba8518dd columnFamilyName B 2024-12-08T00:15:40,903 DEBUG [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:40,903 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(327): Store=7404e7668f405a8889810821ba8518dd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:15:40,903 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,904 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:15:40,904 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7404e7668f405a8889810821ba8518dd columnFamilyName C 2024-12-08T00:15:40,904 DEBUG [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:40,904 INFO [StoreOpener-7404e7668f405a8889810821ba8518dd-1 {}] regionserver.HStore(327): Store=7404e7668f405a8889810821ba8518dd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:15:40,904 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:40,904 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,905 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,906 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,907 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,907 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,907 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:15:40,908 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,909 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 7404e7668f405a8889810821ba8518dd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63258439, jitterRate=-0.05737580358982086}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:15:40,909 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,909 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 7404e7668f405a8889810821ba8518dd: Running coprocessor pre-open hook at 1733616940897Writing region info on filesystem at 1733616940897Initializing all the Stores at 1733616940898 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616940898Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616940898Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616940898Cleaning up temporary data from old regions at 1733616940907 (+9 ms)Running coprocessor post-open hooks at 1733616940909 (+2 ms)Region opened successfully at 1733616940909 2024-12-08T00:15:40,910 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., pid=90, masterSystemTime=1733616940888 2024-12-08T00:15:40,912 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:40,912 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:40,912 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=7404e7668f405a8889810821ba8518dd, regionState=OPEN, openSeqNum=5, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:40,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:15:40,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-12-08T00:15:40,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 in 179 msec 2024-12-08T00:15:40,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-08T00:15:40,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, REOPEN/MOVE in 508 msec 2024-12-08T00:15:40,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-08T00:15:40,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-12-08T00:15:40,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-12-08T00:15:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T00:15:40,922 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:40,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,923 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342af2ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,925 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c21180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eea835b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,928 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f7942a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,929 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25e21627, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,931 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cc616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,932 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700e2633, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,933 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d82666a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,935 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7338989f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,937 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37567063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17400db8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5692727f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,943 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1751e615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,945 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd49c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,947 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@597622e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3b8a6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,951 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb7df86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0000ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:15:40,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:15:40,955 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:15:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:15:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:15:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657ffea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:15:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:15:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:15:40,957 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:15:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5decb70b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:15:40,961 DEBUG [Thread-1353 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,961 DEBUG [Thread-1360 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,961 DEBUG [Thread-1358 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,961 DEBUG [Thread-1347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,961 DEBUG [Thread-1356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,962 DEBUG [Thread-1345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,962 DEBUG [Thread-1349 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,962 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,964 DEBUG [Thread-1364 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,964 DEBUG [Thread-1351 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-08T00:15:40,965 DEBUG [Thread-1362 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:15:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T00:15:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,966 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:15:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:15:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:15:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,969 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:40,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:15:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:40,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T00:15:40,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:40,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:40,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:40,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:40,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:40,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:41,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120841c40de5545c4f23945f5d5595634fac_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616940968/Put/seqid=0 2024-12-08T00:15:41,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617001006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617001007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617001012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742115_1291 (size=14594) 2024-12-08T00:15:41,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733616940988, max timestamp=1733616941015Num uniques 859; 2024-12-08T00:15:41,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T00:15:41,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733616940988, max timestamp=1733616941015Num uniques 859; 2024-12-08T00:15:41,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T00:15:41,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733616940988, max timestamp=1733616941015Num uniques 859; 2024-12-08T00:15:41,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T00:15:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T00:15:41,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617001077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617001079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617001121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617001121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617001121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617001191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617001191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T00:15:41,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617001331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617001331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617001331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617001401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617001401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,423 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:41,426 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120841c40de5545c4f23945f5d5595634fac_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120841c40de5545c4f23945f5d5595634fac_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:41,426 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fb6cf85436644c47a743bb5331497dba, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:41,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fb6cf85436644c47a743bb5331497dba is 175, key is test_row_0/A:col10/1733616940968/Put/seqid=0 2024-12-08T00:15:41,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:41,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:41,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742116_1292 (size=39549) 2024-12-08T00:15:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:41,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T00:15:41,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617001651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617001651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617001652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617001722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617001723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:41,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,833 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fb6cf85436644c47a743bb5331497dba 2024-12-08T00:15:41,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/e93d56eccec34660b4e84dd94f32b55d is 50, key is test_row_0/B:col10/1733616940968/Put/seqid=0 2024-12-08T00:15:41,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:41,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:41,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742117_1293 (size=12001) 2024-12-08T00:15:42,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:42,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T00:15:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617002161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617002161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617002161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:42,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:42,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:42,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617002241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:42,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617002241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:42,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/e93d56eccec34660b4e84dd94f32b55d 2024-12-08T00:15:42,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/be028f2857104140b3c478df97557a14 is 50, key is test_row_0/C:col10/1733616940968/Put/seqid=0 2024-12-08T00:15:42,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742118_1294 (size=12001) 2024-12-08T00:15:42,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:42,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:42,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:42,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:42,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:42,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:42,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/be028f2857104140b3c478df97557a14 2024-12-08T00:15:42,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fb6cf85436644c47a743bb5331497dba as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fb6cf85436644c47a743bb5331497dba 2024-12-08T00:15:42,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fb6cf85436644c47a743bb5331497dba, entries=200, sequenceid=16, filesize=38.6 K 2024-12-08T00:15:42,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/e93d56eccec34660b4e84dd94f32b55d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/e93d56eccec34660b4e84dd94f32b55d 2024-12-08T00:15:42,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/e93d56eccec34660b4e84dd94f32b55d, entries=150, sequenceid=16, filesize=11.7 K 2024-12-08T00:15:42,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/be028f2857104140b3c478df97557a14 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/be028f2857104140b3c478df97557a14 2024-12-08T00:15:42,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/be028f2857104140b3c478df97557a14, entries=150, sequenceid=16, filesize=11.7 K 2024-12-08T00:15:42,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=119.58 KB/122448 for 7404e7668f405a8889810821ba8518dd in 1793ms, sequenceid=16, compaction requested=false 2024-12-08T00:15:42,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:42,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-08T00:15:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:42,814 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=119.58 KB heapSize=342.40 KB 2024-12-08T00:15:42,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:42,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:42,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:42,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:42,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:42,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083abb04bbbbda4b3e91b067718790950c_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616941066/Put/seqid=0 2024-12-08T00:15:42,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742119_1295 (size=12154) 2024-12-08T00:15:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T00:15:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:43,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:43,189 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T00:15:43,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617003193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617003195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617003196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:43,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617003251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,257 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 7404e7668f405a8889810821ba8518dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,257 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083abb04bbbbda4b3e91b067718790950c_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083abb04bbbbda4b3e91b067718790950c_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:43,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/76925bbaa8764652abc3531fbb4020d8, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/76925bbaa8764652abc3531fbb4020d8 is 175, key is test_row_0/A:col10/1733616941066/Put/seqid=0 2024-12-08T00:15:43,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742120_1296 (size=30955) 2024-12-08T00:15:43,263 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=39.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/76925bbaa8764652abc3531fbb4020d8 2024-12-08T00:15:43,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617003261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/fccdb3ffd501482abe3974cd41c08b6e is 50, key is test_row_0/B:col10/1733616941066/Put/seqid=0 2024-12-08T00:15:43,271 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 7404e7668f405a8889810821ba8518dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742121_1297 (size=12001) 2024-12-08T00:15:43,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617003300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617003310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617003311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617003510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617003520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617003520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,677 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/fccdb3ffd501482abe3974cd41c08b6e 2024-12-08T00:15:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/e2c117f84cab438cae9aa29795215b2d is 50, key is test_row_0/C:col10/1733616941066/Put/seqid=0 2024-12-08T00:15:43,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742122_1298 (size=12001) 2024-12-08T00:15:43,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617003821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617003831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617003831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,088 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/e2c117f84cab438cae9aa29795215b2d 2024-12-08T00:15:44,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/76925bbaa8764652abc3531fbb4020d8 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/76925bbaa8764652abc3531fbb4020d8 2024-12-08T00:15:44,097 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/76925bbaa8764652abc3531fbb4020d8, entries=150, sequenceid=51, filesize=30.2 K 2024-12-08T00:15:44,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/fccdb3ffd501482abe3974cd41c08b6e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/fccdb3ffd501482abe3974cd41c08b6e 2024-12-08T00:15:44,103 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/fccdb3ffd501482abe3974cd41c08b6e, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T00:15:44,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/e2c117f84cab438cae9aa29795215b2d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e2c117f84cab438cae9aa29795215b2d 2024-12-08T00:15:44,107 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e2c117f84cab438cae9aa29795215b2d, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T00:15:44,108 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~119.58 KB/122448, heapSize ~342.35 KB/350568, currentSize=60.38 KB/61830 for 7404e7668f405a8889810821ba8518dd in 1295ms, sequenceid=51, compaction requested=false 2024-12-08T00:15:44,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:44,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:44,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-08T00:15:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-08T00:15:44,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-08T00:15:44,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1410 sec 2024-12-08T00:15:44,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 3.1490 sec 2024-12-08T00:15:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:44,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:15:44,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:44,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:44,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:44,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:44,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:44,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:44,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f031cc22cfa04bfba36361125e6e1536_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616943194/Put/seqid=0 2024-12-08T00:15:44,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742123_1299 (size=12154) 2024-12-08T00:15:44,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617004373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:44,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617004374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617004375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617004481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617004481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617004491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617004691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:44,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617004691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617004700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:44,746 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:44,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f031cc22cfa04bfba36361125e6e1536_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f031cc22cfa04bfba36361125e6e1536_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:44,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6efaa04cf75b4800a7dcfdc9af6a38b7, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:44,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6efaa04cf75b4800a7dcfdc9af6a38b7 is 175, key is test_row_0/A:col10/1733616943194/Put/seqid=0 2024-12-08T00:15:44,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742124_1300 (size=30955) 2024-12-08T00:15:45,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617005001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617005001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:45,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617005011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T00:15:45,141 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:45,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-08T00:15:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-08T00:15:45,143 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:45,144 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:45,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:45,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6efaa04cf75b4800a7dcfdc9af6a38b7 2024-12-08T00:15:45,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/6338b3d4733a4a86bbe2dac705468213 is 50, key is test_row_0/B:col10/1733616943194/Put/seqid=0 2024-12-08T00:15:45,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742125_1301 (size=12001) 2024-12-08T00:15:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-08T00:15:45,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617005270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:45,274 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 7404e7668f405a8889810821ba8518dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617005281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:45,286 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 7404e7668f405a8889810821ba8518dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:45,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-08T00:15:45,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:45,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-08T00:15:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-08T00:15:45,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617005510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:45,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617005510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617005531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:45,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/6338b3d4733a4a86bbe2dac705468213 2024-12-08T00:15:45,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/6b938e68f1b34da794d43f6779bbd270 is 50, key is test_row_0/C:col10/1733616943194/Put/seqid=0 2024-12-08T00:15:45,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742126_1302 (size=12001) 2024-12-08T00:15:45,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-08T00:15:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-08T00:15:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-08T00:15:45,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-08T00:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:45,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:45,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/6b938e68f1b34da794d43f6779bbd270 2024-12-08T00:15:45,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6efaa04cf75b4800a7dcfdc9af6a38b7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6efaa04cf75b4800a7dcfdc9af6a38b7 2024-12-08T00:15:45,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6efaa04cf75b4800a7dcfdc9af6a38b7, entries=150, sequenceid=64, filesize=30.2 K 2024-12-08T00:15:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/6338b3d4733a4a86bbe2dac705468213 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6338b3d4733a4a86bbe2dac705468213 2024-12-08T00:15:45,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6338b3d4733a4a86bbe2dac705468213, entries=150, sequenceid=64, filesize=11.7 K 2024-12-08T00:15:45,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/6b938e68f1b34da794d43f6779bbd270 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6b938e68f1b34da794d43f6779bbd270 2024-12-08T00:15:46,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6b938e68f1b34da794d43f6779bbd270, entries=150, sequenceid=64, filesize=11.7 K 2024-12-08T00:15:46,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7404e7668f405a8889810821ba8518dd in 1668ms, sequenceid=64, compaction requested=true 2024-12-08T00:15:46,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:46,003 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:46,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:46,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:46,003 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:46,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:46,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:46,004 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:46,004 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:46,004 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/B is initiating minor compaction (all files) 2024-12-08T00:15:46,004 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/A is initiating minor compaction (all files) 2024-12-08T00:15:46,004 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/A in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:46,004 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/B in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:46,004 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/e93d56eccec34660b4e84dd94f32b55d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/fccdb3ffd501482abe3974cd41c08b6e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6338b3d4733a4a86bbe2dac705468213] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=35.2 K 2024-12-08T00:15:46,004 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fb6cf85436644c47a743bb5331497dba, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/76925bbaa8764652abc3531fbb4020d8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6efaa04cf75b4800a7dcfdc9af6a38b7] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=99.1 K 2024-12-08T00:15:46,004 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:46,004 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fb6cf85436644c47a743bb5331497dba, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/76925bbaa8764652abc3531fbb4020d8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6efaa04cf75b4800a7dcfdc9af6a38b7] 2024-12-08T00:15:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:46,005 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e93d56eccec34660b4e84dd94f32b55d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733616940968 2024-12-08T00:15:46,005 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb6cf85436644c47a743bb5331497dba, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733616940968 2024-12-08T00:15:46,005 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting fccdb3ffd501482abe3974cd41c08b6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733616941066 2024-12-08T00:15:46,005 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76925bbaa8764652abc3531fbb4020d8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733616941066 2024-12-08T00:15:46,005 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6338b3d4733a4a86bbe2dac705468213, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733616943193 2024-12-08T00:15:46,005 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6efaa04cf75b4800a7dcfdc9af6a38b7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733616943193 2024-12-08T00:15:46,012 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:46,013 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/6e0c6c534b1947e39a83260f5fafeba5 is 50, key is test_row_0/B:col10/1733616943194/Put/seqid=0 2024-12-08T00:15:46,015 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:46,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742127_1303 (size=12104) 2024-12-08T00:15:46,025 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120865fdd66a0d6b4f859ae98a3858f98da7_7404e7668f405a8889810821ba8518dd store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:46,026 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120865fdd66a0d6b4f859ae98a3858f98da7_7404e7668f405a8889810821ba8518dd, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:46,026 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120865fdd66a0d6b4f859ae98a3858f98da7_7404e7668f405a8889810821ba8518dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:46,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742128_1304 (size=4469) 2024-12-08T00:15:46,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-08T00:15:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:46,059 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T00:15:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:46,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089a4f3f303e204f2b91a514cd1fea9c0c_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616944372/Put/seqid=0 2024-12-08T00:15:46,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742129_1305 (size=12154) 2024-12-08T00:15:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:46,074 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089a4f3f303e204f2b91a514cd1fea9c0c_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089a4f3f303e204f2b91a514cd1fea9c0c_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:46,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/53e8d907f393467eb9c2fa916c0b2727, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/53e8d907f393467eb9c2fa916c0b2727 is 175, key is test_row_0/A:col10/1733616944372/Put/seqid=0 2024-12-08T00:15:46,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742130_1306 (size=30955) 2024-12-08T00:15:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-08T00:15:46,428 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/6e0c6c534b1947e39a83260f5fafeba5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6e0c6c534b1947e39a83260f5fafeba5 2024-12-08T00:15:46,433 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/B of 7404e7668f405a8889810821ba8518dd into 6e0c6c534b1947e39a83260f5fafeba5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:46,433 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:46,433 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/B, priority=13, startTime=1733616946003; duration=0sec 2024-12-08T00:15:46,433 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:46,433 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:B 2024-12-08T00:15:46,433 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:46,434 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:46,434 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/C is initiating minor compaction (all files) 2024-12-08T00:15:46,434 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/C in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:46,434 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/be028f2857104140b3c478df97557a14, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e2c117f84cab438cae9aa29795215b2d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6b938e68f1b34da794d43f6779bbd270] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=35.2 K 2024-12-08T00:15:46,435 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting be028f2857104140b3c478df97557a14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733616940968 2024-12-08T00:15:46,435 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e2c117f84cab438cae9aa29795215b2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733616941066 2024-12-08T00:15:46,435 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b938e68f1b34da794d43f6779bbd270, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733616943193 2024-12-08T00:15:46,441 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:46,441 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/3d01a0e0403549a5aa8cfcc565302571 is 50, key is test_row_0/C:col10/1733616943194/Put/seqid=0 2024-12-08T00:15:46,443 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#A#compaction#256 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:46,444 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/b8ea19b694684bfa80fe32e39f3a0013 is 175, key is test_row_0/A:col10/1733616943194/Put/seqid=0 2024-12-08T00:15:46,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742131_1307 (size=12104) 2024-12-08T00:15:46,452 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/3d01a0e0403549a5aa8cfcc565302571 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3d01a0e0403549a5aa8cfcc565302571 2024-12-08T00:15:46,458 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/C of 7404e7668f405a8889810821ba8518dd into 3d01a0e0403549a5aa8cfcc565302571(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:46,458 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:46,458 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/C, priority=13, startTime=1733616946004; duration=0sec 2024-12-08T00:15:46,458 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:46,458 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:C 2024-12-08T00:15:46,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742132_1308 (size=31058) 2024-12-08T00:15:46,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/53e8d907f393467eb9c2fa916c0b2727 2024-12-08T00:15:46,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/4cf2031bf73945b58aabf9ade83abf3c is 50, key is test_row_0/B:col10/1733616944372/Put/seqid=0 2024-12-08T00:15:46,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742133_1309 (size=12001) 2024-12-08T00:15:46,510 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/4cf2031bf73945b58aabf9ade83abf3c 2024-12-08T00:15:46,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/268a7dd605044630945dd9bf85f9b707 is 50, key is test_row_0/C:col10/1733616944372/Put/seqid=0 2024-12-08T00:15:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:46,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:46,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742134_1310 (size=12001) 2024-12-08T00:15:46,533 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/268a7dd605044630945dd9bf85f9b707 2024-12-08T00:15:46,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/53e8d907f393467eb9c2fa916c0b2727 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/53e8d907f393467eb9c2fa916c0b2727 2024-12-08T00:15:46,543 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/53e8d907f393467eb9c2fa916c0b2727, entries=150, sequenceid=83, filesize=30.2 K 2024-12-08T00:15:46,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/4cf2031bf73945b58aabf9ade83abf3c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/4cf2031bf73945b58aabf9ade83abf3c 2024-12-08T00:15:46,552 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/4cf2031bf73945b58aabf9ade83abf3c, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T00:15:46,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/268a7dd605044630945dd9bf85f9b707 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/268a7dd605044630945dd9bf85f9b707 2024-12-08T00:15:46,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617006545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:46,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617006551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:46,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617006552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:46,558 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 7404e7668f405a8889810821ba8518dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:46,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:46,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:46,561 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/268a7dd605044630945dd9bf85f9b707, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T00:15:46,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7404e7668f405a8889810821ba8518dd in 503ms, sequenceid=83, compaction requested=false 2024-12-08T00:15:46,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:46,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:46,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-08T00:15:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-08T00:15:46,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-08T00:15:46,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-12-08T00:15:46,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.4240 sec 2024-12-08T00:15:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:46,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T00:15:46,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:46,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:46,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:46,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:46,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:46,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:46,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120893b65cb3254e4f33b2ef9f8b501d7f49_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616946661/Put/seqid=0 2024-12-08T00:15:46,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742135_1311 (size=14594) 2024-12-08T00:15:46,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:46,696 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120893b65cb3254e4f33b2ef9f8b501d7f49_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120893b65cb3254e4f33b2ef9f8b501d7f49_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:46,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/f76997f83fbe4e788be77926fab0de6f, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:46,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/f76997f83fbe4e788be77926fab0de6f is 175, key is test_row_0/A:col10/1733616946661/Put/seqid=0 2024-12-08T00:15:46,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742136_1312 (size=39549) 2024-12-08T00:15:46,710 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/f76997f83fbe4e788be77926fab0de6f 2024-12-08T00:15:46,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/49ff60bd608449c2a0480fd76e134781 is 50, key is test_row_0/B:col10/1733616946661/Put/seqid=0 2024-12-08T00:15:46,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742137_1313 (size=12001) 2024-12-08T00:15:46,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617006745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:46,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617006746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:46,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617006861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:46,866 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/b8ea19b694684bfa80fe32e39f3a0013 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/b8ea19b694684bfa80fe32e39f3a0013 2024-12-08T00:15:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:46,870 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/A of 7404e7668f405a8889810821ba8518dd into b8ea19b694684bfa80fe32e39f3a0013(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:46,870 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:46,870 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/A, priority=13, startTime=1733616946002; duration=0sec 2024-12-08T00:15:46,870 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:46,870 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:A 2024-12-08T00:15:46,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617006870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:47,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617007081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617007081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:47,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/49ff60bd608449c2a0480fd76e134781 2024-12-08T00:15:47,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/e48408cbc3234d979ec6781aabaae06d is 50, key is test_row_0/C:col10/1733616946661/Put/seqid=0 2024-12-08T00:15:47,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742138_1314 (size=12001) 2024-12-08T00:15:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-08T00:15:47,291 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:47,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-08T00:15:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T00:15:47,297 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:47,298 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:47,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T00:15:47,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617007391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617007391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:47,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T00:15:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:47,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:47,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/e48408cbc3234d979ec6781aabaae06d 2024-12-08T00:15:47,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/f76997f83fbe4e788be77926fab0de6f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f76997f83fbe4e788be77926fab0de6f 2024-12-08T00:15:47,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f76997f83fbe4e788be77926fab0de6f, entries=200, sequenceid=99, filesize=38.6 K 2024-12-08T00:15:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/49ff60bd608449c2a0480fd76e134781 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/49ff60bd608449c2a0480fd76e134781 2024-12-08T00:15:47,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/49ff60bd608449c2a0480fd76e134781, entries=150, sequenceid=99, filesize=11.7 K 2024-12-08T00:15:47,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/e48408cbc3234d979ec6781aabaae06d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e48408cbc3234d979ec6781aabaae06d 2024-12-08T00:15:47,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e48408cbc3234d979ec6781aabaae06d, entries=150, sequenceid=99, filesize=11.7 K 2024-12-08T00:15:47,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7404e7668f405a8889810821ba8518dd in 900ms, sequenceid=99, compaction requested=true 2024-12-08T00:15:47,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:47,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:47,563 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:47,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:47,563 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:47,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:47,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:47,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:47,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:47,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:47,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:47,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/B is initiating minor compaction (all files) 2024-12-08T00:15:47,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/A is initiating minor compaction (all files) 2024-12-08T00:15:47,569 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/B in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:47,569 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/A in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:47,569 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/b8ea19b694684bfa80fe32e39f3a0013, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/53e8d907f393467eb9c2fa916c0b2727, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f76997f83fbe4e788be77926fab0de6f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=99.2 K 2024-12-08T00:15:47,569 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6e0c6c534b1947e39a83260f5fafeba5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/4cf2031bf73945b58aabf9ade83abf3c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/49ff60bd608449c2a0480fd76e134781] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=35.3 K 2024-12-08T00:15:47,569 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:47,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/b8ea19b694684bfa80fe32e39f3a0013, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/53e8d907f393467eb9c2fa916c0b2727, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f76997f83fbe4e788be77926fab0de6f] 2024-12-08T00:15:47,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0c6c534b1947e39a83260f5fafeba5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733616943193 2024-12-08T00:15:47,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8ea19b694684bfa80fe32e39f3a0013, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733616943193 2024-12-08T00:15:47,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53e8d907f393467eb9c2fa916c0b2727, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733616944364 2024-12-08T00:15:47,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf2031bf73945b58aabf9ade83abf3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733616944364 2024-12-08T00:15:47,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 49ff60bd608449c2a0480fd76e134781, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733616946538 2024-12-08T00:15:47,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting f76997f83fbe4e788be77926fab0de6f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733616946538 2024-12-08T00:15:47,577 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:47,578 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:47,579 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/7e219ba0c9184004b8f3ea72e3c389f6 is 50, key is test_row_0/B:col10/1733616946661/Put/seqid=0 2024-12-08T00:15:47,581 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208769b8fef5e4b4fb98684e5cbb6b12169_7404e7668f405a8889810821ba8518dd store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:47,583 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208769b8fef5e4b4fb98684e5cbb6b12169_7404e7668f405a8889810821ba8518dd, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:47,583 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208769b8fef5e4b4fb98684e5cbb6b12169_7404e7668f405a8889810821ba8518dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:47,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742139_1315 (size=12207) 2024-12-08T00:15:47,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742140_1316 (size=4469) 2024-12-08T00:15:47,598 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/7e219ba0c9184004b8f3ea72e3c389f6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/7e219ba0c9184004b8f3ea72e3c389f6 2024-12-08T00:15:47,604 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/B of 7404e7668f405a8889810821ba8518dd into 7e219ba0c9184004b8f3ea72e3c389f6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:47,604 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:47,604 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/B, priority=13, startTime=1733616947563; duration=0sec 2024-12-08T00:15:47,604 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:47,604 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:B 2024-12-08T00:15:47,605 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:47,605 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:47,605 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/C is initiating minor compaction (all files) 2024-12-08T00:15:47,605 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/C in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:47,606 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3d01a0e0403549a5aa8cfcc565302571, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/268a7dd605044630945dd9bf85f9b707, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e48408cbc3234d979ec6781aabaae06d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=35.3 K 2024-12-08T00:15:47,606 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d01a0e0403549a5aa8cfcc565302571, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733616943193 2024-12-08T00:15:47,606 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 268a7dd605044630945dd9bf85f9b707, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733616944364 2024-12-08T00:15:47,606 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e48408cbc3234d979ec6781aabaae06d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733616946538 2024-12-08T00:15:47,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T00:15:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:47,607 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T00:15:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T00:15:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208297af6670cca4754bca7a8bb20fe2b4e_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616946711/Put/seqid=0 2024-12-08T00:15:47,616 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:47,617 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/6bd97964636546c3abea1e59d320b734 is 50, key is test_row_0/C:col10/1733616946661/Put/seqid=0 2024-12-08T00:15:47,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742141_1317 (size=12154) 2024-12-08T00:15:47,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:47,625 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208297af6670cca4754bca7a8bb20fe2b4e_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208297af6670cca4754bca7a8bb20fe2b4e_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/4fb3e4af23e34dc7a474aea7d49d71c7, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/4fb3e4af23e34dc7a474aea7d49d71c7 is 175, key is test_row_0/A:col10/1733616946711/Put/seqid=0 2024-12-08T00:15:47,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742143_1319 (size=30955) 2024-12-08T00:15:47,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742142_1318 (size=12207) 2024-12-08T00:15:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:47,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T00:15:47,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617007942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:47,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617007952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:47,990 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#A#compaction#265 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:47,991 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/10812f38e9234a45a73f8d41c4a4f4cd is 175, key is test_row_0/A:col10/1733616946661/Put/seqid=0 2024-12-08T00:15:47,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742144_1320 (size=31161) 2024-12-08T00:15:48,031 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/4fb3e4af23e34dc7a474aea7d49d71c7 2024-12-08T00:15:48,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/ffad9704288d4bb5bfabba1c01d71c3c is 50, key is test_row_0/B:col10/1733616946711/Put/seqid=0 2024-12-08T00:15:48,041 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/6bd97964636546c3abea1e59d320b734 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6bd97964636546c3abea1e59d320b734 2024-12-08T00:15:48,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742145_1321 (size=12001) 2024-12-08T00:15:48,045 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/C of 7404e7668f405a8889810821ba8518dd into 6bd97964636546c3abea1e59d320b734(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:48,045 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:48,045 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/C, priority=13, startTime=1733616947564; duration=0sec 2024-12-08T00:15:48,046 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:48,046 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:C 2024-12-08T00:15:48,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617008060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:48,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617008071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:48,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617008271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:48,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617008280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T00:15:48,433 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/10812f38e9234a45a73f8d41c4a4f4cd as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/10812f38e9234a45a73f8d41c4a4f4cd 2024-12-08T00:15:48,438 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/A of 7404e7668f405a8889810821ba8518dd into 10812f38e9234a45a73f8d41c4a4f4cd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:48,438 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:48,438 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/A, priority=13, startTime=1733616947563; duration=0sec 2024-12-08T00:15:48,438 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:48,438 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:A 2024-12-08T00:15:48,445 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/ffad9704288d4bb5bfabba1c01d71c3c 2024-12-08T00:15:48,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/b1c6d95c96304b77b8c881cf451d7acc is 50, key is test_row_0/C:col10/1733616946711/Put/seqid=0 2024-12-08T00:15:48,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742146_1322 (size=12001) 2024-12-08T00:15:48,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617008571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:48,575 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 7404e7668f405a8889810821ba8518dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:48,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617008581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:48,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617008591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:48,855 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/b1c6d95c96304b77b8c881cf451d7acc 2024-12-08T00:15:48,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/4fb3e4af23e34dc7a474aea7d49d71c7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/4fb3e4af23e34dc7a474aea7d49d71c7 2024-12-08T00:15:48,863 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/4fb3e4af23e34dc7a474aea7d49d71c7, entries=150, sequenceid=119, filesize=30.2 K 2024-12-08T00:15:48,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/ffad9704288d4bb5bfabba1c01d71c3c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ffad9704288d4bb5bfabba1c01d71c3c 2024-12-08T00:15:48,866 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ffad9704288d4bb5bfabba1c01d71c3c, entries=150, sequenceid=119, filesize=11.7 K 2024-12-08T00:15:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/b1c6d95c96304b77b8c881cf451d7acc as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b1c6d95c96304b77b8c881cf451d7acc 2024-12-08T00:15:48,870 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b1c6d95c96304b77b8c881cf451d7acc, entries=150, sequenceid=119, filesize=11.7 K 2024-12-08T00:15:48,871 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7404e7668f405a8889810821ba8518dd in 1263ms, sequenceid=119, compaction requested=false 2024-12-08T00:15:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-08T00:15:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-08T00:15:48,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-08T00:15:48,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2024-12-08T00:15:48,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.5810 sec 2024-12-08T00:15:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:49,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T00:15:49,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:49,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:49,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:49,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:49,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:49,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:49,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081cc31e81dbc1480a8f67c719c67b0b89_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616947926/Put/seqid=0 2024-12-08T00:15:49,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742147_1323 (size=14744) 2024-12-08T00:15:49,108 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:49,111 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081cc31e81dbc1480a8f67c719c67b0b89_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081cc31e81dbc1480a8f67c719c67b0b89_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:49,112 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/aefba380d4634aefaa3d5f562c0404ea, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:49,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/aefba380d4634aefaa3d5f562c0404ea is 175, key is test_row_0/A:col10/1733616947926/Put/seqid=0 2024-12-08T00:15:49,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742148_1324 (size=39699) 2024-12-08T00:15:49,126 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/aefba380d4634aefaa3d5f562c0404ea 2024-12-08T00:15:49,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/549a3dc7fa4b4be7a9654b709cc1eb84 is 50, key is test_row_0/B:col10/1733616947926/Put/seqid=0 2024-12-08T00:15:49,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742149_1325 (size=12101) 2024-12-08T00:15:49,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617009161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617009162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617009271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617009271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617009301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,303 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 7404e7668f405a8889810821ba8518dd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617009311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,312 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 7404e7668f405a8889810821ba8518dd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T00:15:49,441 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:49,441 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-08T00:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T00:15:49,443 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:49,443 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:49,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:49,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617009480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617009480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/549a3dc7fa4b4be7a9654b709cc1eb84 2024-12-08T00:15:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T00:15:49,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/9dac6461cbb14f84a997a0270fae3e25 is 50, key is test_row_0/C:col10/1733616947926/Put/seqid=0 2024-12-08T00:15:49,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742150_1326 (size=12101) 2024-12-08T00:15:49,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/9dac6461cbb14f84a997a0270fae3e25 2024-12-08T00:15:49,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/aefba380d4634aefaa3d5f562c0404ea as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/aefba380d4634aefaa3d5f562c0404ea 2024-12-08T00:15:49,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/aefba380d4634aefaa3d5f562c0404ea, entries=200, sequenceid=136, filesize=38.8 K 2024-12-08T00:15:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/549a3dc7fa4b4be7a9654b709cc1eb84 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/549a3dc7fa4b4be7a9654b709cc1eb84 2024-12-08T00:15:49,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/549a3dc7fa4b4be7a9654b709cc1eb84, entries=150, sequenceid=136, filesize=11.8 K 2024-12-08T00:15:49,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/9dac6461cbb14f84a997a0270fae3e25 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9dac6461cbb14f84a997a0270fae3e25 2024-12-08T00:15:49,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9dac6461cbb14f84a997a0270fae3e25, entries=150, sequenceid=136, filesize=11.8 K 2024-12-08T00:15:49,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 7404e7668f405a8889810821ba8518dd in 481ms, sequenceid=136, compaction requested=true 2024-12-08T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:49,576 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:49,576 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:49,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:49,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/A is initiating minor compaction (all files) 2024-12-08T00:15:49,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:49,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/B is initiating minor compaction (all files) 2024-12-08T00:15:49,577 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/A in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:49,577 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/B in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:49,577 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/10812f38e9234a45a73f8d41c4a4f4cd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/4fb3e4af23e34dc7a474aea7d49d71c7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/aefba380d4634aefaa3d5f562c0404ea] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=99.4 K 2024-12-08T00:15:49,577 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/7e219ba0c9184004b8f3ea72e3c389f6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ffad9704288d4bb5bfabba1c01d71c3c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/549a3dc7fa4b4be7a9654b709cc1eb84] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=35.5 K 2024-12-08T00:15:49,577 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:49,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/10812f38e9234a45a73f8d41c4a4f4cd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/4fb3e4af23e34dc7a474aea7d49d71c7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/aefba380d4634aefaa3d5f562c0404ea] 2024-12-08T00:15:49,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e219ba0c9184004b8f3ea72e3c389f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733616946538 2024-12-08T00:15:49,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10812f38e9234a45a73f8d41c4a4f4cd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733616946538 2024-12-08T00:15:49,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ffad9704288d4bb5bfabba1c01d71c3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733616946711 2024-12-08T00:15:49,578 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fb3e4af23e34dc7a474aea7d49d71c7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733616946711 2024-12-08T00:15:49,578 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 549a3dc7fa4b4be7a9654b709cc1eb84, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733616947926 2024-12-08T00:15:49,578 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting aefba380d4634aefaa3d5f562c0404ea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733616947926 2024-12-08T00:15:49,582 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:49,584 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:49,585 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/bf39b5a4125346449e4278f45d22f257 is 50, key is test_row_0/B:col10/1733616947926/Put/seqid=0 2024-12-08T00:15:49,586 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208c5a0bca2d43449d68803ccc7e7a98fa2_7404e7668f405a8889810821ba8518dd store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:49,588 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208c5a0bca2d43449d68803ccc7e7a98fa2_7404e7668f405a8889810821ba8518dd, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:49,588 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c5a0bca2d43449d68803ccc7e7a98fa2_7404e7668f405a8889810821ba8518dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:49,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T00:15:49,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:49,596 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T00:15:49,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:49,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:49,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:49,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:49,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:49,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:49,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742151_1327 (size=12409) 2024-12-08T00:15:49,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742152_1328 (size=4469) 2024-12-08T00:15:49,611 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#A#compaction#273 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:49,612 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/2bbdb3bccfbc4422a4835fdbcec825cc is 175, key is test_row_0/A:col10/1733616947926/Put/seqid=0 2024-12-08T00:15:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208577854ba23434f1db04681af550dbb24_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616949137/Put/seqid=0 2024-12-08T00:15:49,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742153_1329 (size=31363) 2024-12-08T00:15:49,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742154_1330 (size=12304) 2024-12-08T00:15:49,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:49,631 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208577854ba23434f1db04681af550dbb24_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208577854ba23434f1db04681af550dbb24_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/0bc715b24d2d4f85b21e360d7b1c2df5, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/0bc715b24d2d4f85b21e360d7b1c2df5 is 175, key is test_row_0/A:col10/1733616949137/Put/seqid=0 2024-12-08T00:15:49,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742155_1331 (size=31105) 2024-12-08T00:15:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T00:15:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:49,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617009826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617009830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617009940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617009951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:50,001 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/bf39b5a4125346449e4278f45d22f257 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/bf39b5a4125346449e4278f45d22f257 2024-12-08T00:15:50,005 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/B of 7404e7668f405a8889810821ba8518dd into bf39b5a4125346449e4278f45d22f257(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:50,005 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:50,005 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/B, priority=13, startTime=1733616949576; duration=0sec 2024-12-08T00:15:50,006 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:50,006 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:B 2024-12-08T00:15:50,006 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:50,007 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:50,007 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/C is initiating minor compaction (all files) 2024-12-08T00:15:50,007 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/C in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:50,007 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6bd97964636546c3abea1e59d320b734, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b1c6d95c96304b77b8c881cf451d7acc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9dac6461cbb14f84a997a0270fae3e25] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=35.5 K 2024-12-08T00:15:50,007 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bd97964636546c3abea1e59d320b734, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733616946538 2024-12-08T00:15:50,007 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c6d95c96304b77b8c881cf451d7acc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733616946711 2024-12-08T00:15:50,007 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dac6461cbb14f84a997a0270fae3e25, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733616947926 2024-12-08T00:15:50,013 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#C#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:50,013 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/826c0365f9a34a519c83e04057753b88 is 50, key is test_row_0/C:col10/1733616947926/Put/seqid=0 2024-12-08T00:15:50,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742156_1332 (size=12409) 2024-12-08T00:15:50,029 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/2bbdb3bccfbc4422a4835fdbcec825cc as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/2bbdb3bccfbc4422a4835fdbcec825cc 2024-12-08T00:15:50,033 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/A of 7404e7668f405a8889810821ba8518dd into 2bbdb3bccfbc4422a4835fdbcec825cc(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:50,033 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:50,033 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/A, priority=13, startTime=1733616949576; duration=0sec 2024-12-08T00:15:50,034 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:50,034 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:A 2024-12-08T00:15:50,036 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/0bc715b24d2d4f85b21e360d7b1c2df5 2024-12-08T00:15:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/a867a8cda14042d7a91d0fe042181cb5 is 50, key is test_row_0/B:col10/1733616949137/Put/seqid=0 2024-12-08T00:15:50,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742157_1333 (size=12151) 2024-12-08T00:15:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T00:15:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617010150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:50,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617010171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:50,420 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/826c0365f9a34a519c83e04057753b88 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/826c0365f9a34a519c83e04057753b88 2024-12-08T00:15:50,425 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/C of 7404e7668f405a8889810821ba8518dd into 826c0365f9a34a519c83e04057753b88(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:50,425 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:50,425 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/C, priority=13, startTime=1733616949576; duration=0sec 2024-12-08T00:15:50,425 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:50,425 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:C 2024-12-08T00:15:50,446 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/a867a8cda14042d7a91d0fe042181cb5 2024-12-08T00:15:50,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/1fe067f1abf2463d88a169a06b19385a is 50, key is test_row_0/C:col10/1733616949137/Put/seqid=0 2024-12-08T00:15:50,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742158_1334 (size=12151) 2024-12-08T00:15:50,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617010461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:50,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617010481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T00:15:50,857 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/1fe067f1abf2463d88a169a06b19385a 2024-12-08T00:15:50,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/0bc715b24d2d4f85b21e360d7b1c2df5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0bc715b24d2d4f85b21e360d7b1c2df5 2024-12-08T00:15:50,864 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0bc715b24d2d4f85b21e360d7b1c2df5, entries=150, sequenceid=153, filesize=30.4 K 2024-12-08T00:15:50,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/a867a8cda14042d7a91d0fe042181cb5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/a867a8cda14042d7a91d0fe042181cb5 2024-12-08T00:15:50,868 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/a867a8cda14042d7a91d0fe042181cb5, entries=150, sequenceid=153, filesize=11.9 K 2024-12-08T00:15:50,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/1fe067f1abf2463d88a169a06b19385a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/1fe067f1abf2463d88a169a06b19385a 2024-12-08T00:15:50,872 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/1fe067f1abf2463d88a169a06b19385a, entries=150, sequenceid=153, filesize=11.9 K 2024-12-08T00:15:50,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 7404e7668f405a8889810821ba8518dd in 1277ms, sequenceid=153, compaction requested=false 2024-12-08T00:15:50,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:50,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:50,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-08T00:15:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-08T00:15:50,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-08T00:15:50,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-12-08T00:15:50,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4380 sec 2024-12-08T00:15:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:50,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T00:15:50,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:50,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:50,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:50,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:50,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:50,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:50,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bf46f5383cd6461dafbb0c408ebc01ae_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616950973/Put/seqid=0 2024-12-08T00:15:50,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742159_1335 (size=14794) 2024-12-08T00:15:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617011027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617011034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:51,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617011141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:51,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617011151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:51,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617011360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:51,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617011370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:51,385 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:51,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bf46f5383cd6461dafbb0c408ebc01ae_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bf46f5383cd6461dafbb0c408ebc01ae_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:51,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/d7364370195c48f6a822b80648286305, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:51,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/d7364370195c48f6a822b80648286305 is 175, key is test_row_0/A:col10/1733616950973/Put/seqid=0 2024-12-08T00:15:51,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742160_1336 (size=39749) 2024-12-08T00:15:51,393 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/d7364370195c48f6a822b80648286305 2024-12-08T00:15:51,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/eee6f5dcd7024f68ae968196467cdb37 is 50, key is test_row_0/B:col10/1733616950973/Put/seqid=0 2024-12-08T00:15:51,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742161_1337 (size=12151) 2024-12-08T00:15:51,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/eee6f5dcd7024f68ae968196467cdb37 2024-12-08T00:15:51,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/708f6203ecd04c9bbeda87c1ef348813 is 50, key is test_row_0/C:col10/1733616950973/Put/seqid=0 2024-12-08T00:15:51,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742162_1338 (size=12151) 2024-12-08T00:15:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T00:15:51,591 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:51,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-08T00:15:51,593 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T00:15:51,593 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:51,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:51,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617011671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:51,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617011681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T00:15:51,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T00:15:51,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:51,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:51,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:51,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:51,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:51,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/708f6203ecd04c9bbeda87c1ef348813 2024-12-08T00:15:51,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/d7364370195c48f6a822b80648286305 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d7364370195c48f6a822b80648286305 2024-12-08T00:15:51,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d7364370195c48f6a822b80648286305, entries=200, sequenceid=172, filesize=38.8 K 2024-12-08T00:15:51,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/eee6f5dcd7024f68ae968196467cdb37 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/eee6f5dcd7024f68ae968196467cdb37 2024-12-08T00:15:51,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/eee6f5dcd7024f68ae968196467cdb37, entries=150, sequenceid=172, filesize=11.9 K 2024-12-08T00:15:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/708f6203ecd04c9bbeda87c1ef348813 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/708f6203ecd04c9bbeda87c1ef348813 2024-12-08T00:15:51,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/708f6203ecd04c9bbeda87c1ef348813, entries=150, sequenceid=172, filesize=11.9 K 2024-12-08T00:15:51,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7404e7668f405a8889810821ba8518dd in 870ms, sequenceid=172, compaction requested=true 2024-12-08T00:15:51,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:51,845 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:51,845 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:51,845 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:51,845 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:51,846 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/A is initiating minor compaction (all files) 2024-12-08T00:15:51,846 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/B is initiating minor compaction (all files) 2024-12-08T00:15:51,846 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/A in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:51,846 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/B in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:51,846 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/bf39b5a4125346449e4278f45d22f257, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/a867a8cda14042d7a91d0fe042181cb5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/eee6f5dcd7024f68ae968196467cdb37] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=35.9 K 2024-12-08T00:15:51,846 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/2bbdb3bccfbc4422a4835fdbcec825cc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0bc715b24d2d4f85b21e360d7b1c2df5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d7364370195c48f6a822b80648286305] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=99.8 K 2024-12-08T00:15:51,846 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:51,846 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/2bbdb3bccfbc4422a4835fdbcec825cc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0bc715b24d2d4f85b21e360d7b1c2df5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d7364370195c48f6a822b80648286305] 2024-12-08T00:15:51,846 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting bf39b5a4125346449e4278f45d22f257, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733616947926 2024-12-08T00:15:51,846 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bbdb3bccfbc4422a4835fdbcec825cc, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733616947926 2024-12-08T00:15:51,846 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc715b24d2d4f85b21e360d7b1c2df5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733616949137 2024-12-08T00:15:51,846 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a867a8cda14042d7a91d0fe042181cb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733616949137 2024-12-08T00:15:51,847 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting eee6f5dcd7024f68ae968196467cdb37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733616949824 2024-12-08T00:15:51,847 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7364370195c48f6a822b80648286305, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733616949815 2024-12-08T00:15:51,852 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:51,853 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:51,853 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/23719d7d142447ceba090650cfe6db67 is 50, key is test_row_0/B:col10/1733616950973/Put/seqid=0 2024-12-08T00:15:51,855 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412087d418c782872456c81e78acfb8f4bf23_7404e7668f405a8889810821ba8518dd store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:51,857 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412087d418c782872456c81e78acfb8f4bf23_7404e7668f405a8889810821ba8518dd, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:51,857 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087d418c782872456c81e78acfb8f4bf23_7404e7668f405a8889810821ba8518dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:51,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742164_1340 (size=4469) 2024-12-08T00:15:51,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742163_1339 (size=12561) 2024-12-08T00:15:51,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T00:15:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:51,898 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T00:15:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e6fd6374256f4ee59297e5b8e1ac8b7a_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616951026/Put/seqid=0 2024-12-08T00:15:51,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742165_1341 (size=12304) 2024-12-08T00:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T00:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:52,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T00:15:52,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617012241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617012250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:52,264 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#A#compaction#283 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:52,264 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/1ce05430ffae4839b715cc31a690ab9a is 175, key is test_row_0/A:col10/1733616950973/Put/seqid=0 2024-12-08T00:15:52,271 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/23719d7d142447ceba090650cfe6db67 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/23719d7d142447ceba090650cfe6db67 2024-12-08T00:15:52,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742166_1342 (size=31515) 2024-12-08T00:15:52,288 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/B of 7404e7668f405a8889810821ba8518dd into 23719d7d142447ceba090650cfe6db67(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:52,288 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:52,288 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/B, priority=13, startTime=1733616951845; duration=0sec 2024-12-08T00:15:52,288 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:52,288 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:B 2024-12-08T00:15:52,288 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:52,290 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/1ce05430ffae4839b715cc31a690ab9a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/1ce05430ffae4839b715cc31a690ab9a 2024-12-08T00:15:52,290 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:52,290 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/C is initiating minor compaction (all files) 2024-12-08T00:15:52,290 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/C in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:52,290 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/826c0365f9a34a519c83e04057753b88, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/1fe067f1abf2463d88a169a06b19385a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/708f6203ecd04c9bbeda87c1ef348813] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=35.9 K 2024-12-08T00:15:52,291 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 826c0365f9a34a519c83e04057753b88, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733616947926 2024-12-08T00:15:52,291 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe067f1abf2463d88a169a06b19385a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733616949137 2024-12-08T00:15:52,292 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 708f6203ecd04c9bbeda87c1ef348813, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733616949824 2024-12-08T00:15:52,296 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/A of 7404e7668f405a8889810821ba8518dd into 1ce05430ffae4839b715cc31a690ab9a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:52,296 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:52,296 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/A, priority=13, startTime=1733616951845; duration=0sec 2024-12-08T00:15:52,296 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:52,296 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:A 2024-12-08T00:15:52,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:52,313 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e6fd6374256f4ee59297e5b8e1ac8b7a_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e6fd6374256f4ee59297e5b8e1ac8b7a_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:52,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/f0f8ed576f6e41daa797001dfefecb73, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:52,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/f0f8ed576f6e41daa797001dfefecb73 is 175, key is test_row_0/A:col10/1733616951026/Put/seqid=0 2024-12-08T00:15:52,315 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:52,315 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/fd99674c2b5b42cd88a2d4ea57f4bde7 is 50, key is test_row_0/C:col10/1733616950973/Put/seqid=0 2024-12-08T00:15:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742167_1343 (size=31105) 2024-12-08T00:15:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742168_1344 (size=12561) 2024-12-08T00:15:52,326 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/f0f8ed576f6e41daa797001dfefecb73 2024-12-08T00:15:52,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/8d9038f5767f451fab8abffd77420cbe is 50, key is test_row_0/B:col10/1733616951026/Put/seqid=0 2024-12-08T00:15:52,335 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/fd99674c2b5b42cd88a2d4ea57f4bde7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/fd99674c2b5b42cd88a2d4ea57f4bde7 2024-12-08T00:15:52,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742169_1345 (size=12151) 2024-12-08T00:15:52,336 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/8d9038f5767f451fab8abffd77420cbe 2024-12-08T00:15:52,342 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/C of 7404e7668f405a8889810821ba8518dd into fd99674c2b5b42cd88a2d4ea57f4bde7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:52,342 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:52,342 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/C, priority=13, startTime=1733616951845; duration=0sec 2024-12-08T00:15:52,342 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:52,342 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:C 2024-12-08T00:15:52,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/e13b8fdacc224d21a348ab7e43f5b6b3 is 50, key is test_row_0/C:col10/1733616951026/Put/seqid=0 2024-12-08T00:15:52,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742170_1346 (size=12151) 2024-12-08T00:15:52,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617012361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:52,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617012361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:52,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617012570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617012570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:52,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32796 deadline: 1733617012611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:52,618 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 7404e7668f405a8889810821ba8518dd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:15:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T00:15:52,750 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/e13b8fdacc224d21a348ab7e43f5b6b3 2024-12-08T00:15:52,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/f0f8ed576f6e41daa797001dfefecb73 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f0f8ed576f6e41daa797001dfefecb73 2024-12-08T00:15:52,758 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f0f8ed576f6e41daa797001dfefecb73, entries=150, sequenceid=188, filesize=30.4 K 2024-12-08T00:15:52,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/8d9038f5767f451fab8abffd77420cbe as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/8d9038f5767f451fab8abffd77420cbe 2024-12-08T00:15:52,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/8d9038f5767f451fab8abffd77420cbe, entries=150, sequenceid=188, filesize=11.9 K 2024-12-08T00:15:52,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/e13b8fdacc224d21a348ab7e43f5b6b3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e13b8fdacc224d21a348ab7e43f5b6b3 2024-12-08T00:15:52,768 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e13b8fdacc224d21a348ab7e43f5b6b3, entries=150, sequenceid=188, filesize=11.9 K 2024-12-08T00:15:52,769 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7404e7668f405a8889810821ba8518dd in 871ms, sequenceid=188, compaction requested=false 2024-12-08T00:15:52,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:52,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:52,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-08T00:15:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-08T00:15:52,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-08T00:15:52,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1770 sec 2024-12-08T00:15:52,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.1800 sec 2024-12-08T00:15:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:52,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T00:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:52,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080aa55936f2ff4e0a8f2fe699a412708b_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616952882/Put/seqid=0 2024-12-08T00:15:52,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742171_1347 (size=14794) 2024-12-08T00:15:52,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617012927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617012927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:53,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617013040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:53,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617013041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:53,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617013261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:53,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617013261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:53,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:53,296 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080aa55936f2ff4e0a8f2fe699a412708b_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080aa55936f2ff4e0a8f2fe699a412708b_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:53,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/13df0183b38145879c92a34f5223796d, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:53,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/13df0183b38145879c92a34f5223796d is 175, key is test_row_0/A:col10/1733616952882/Put/seqid=0 2024-12-08T00:15:53,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742172_1348 (size=39749) 2024-12-08T00:15:53,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617013571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617013581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:53,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/13df0183b38145879c92a34f5223796d 2024-12-08T00:15:53,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/c21206e878f64937aab1680587f00ece is 50, key is test_row_0/B:col10/1733616952882/Put/seqid=0 2024-12-08T00:15:53,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742173_1349 (size=12151) 2024-12-08T00:15:53,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/c21206e878f64937aab1680587f00ece 2024-12-08T00:15:53,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/3b698d86a71046fe84bd969253271289 is 50, key is test_row_0/C:col10/1733616952882/Put/seqid=0 2024-12-08T00:15:53,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742174_1350 (size=12151) 2024-12-08T00:15:53,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/3b698d86a71046fe84bd969253271289 2024-12-08T00:15:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T00:15:53,741 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:53,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-08T00:15:53,744 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T00:15:53,744 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:53,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/13df0183b38145879c92a34f5223796d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/13df0183b38145879c92a34f5223796d 2024-12-08T00:15:53,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:53,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/13df0183b38145879c92a34f5223796d, entries=200, sequenceid=208, filesize=38.8 K 2024-12-08T00:15:53,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/c21206e878f64937aab1680587f00ece as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/c21206e878f64937aab1680587f00ece 2024-12-08T00:15:53,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/c21206e878f64937aab1680587f00ece, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T00:15:53,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/3b698d86a71046fe84bd969253271289 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3b698d86a71046fe84bd969253271289 2024-12-08T00:15:53,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3b698d86a71046fe84bd969253271289, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T00:15:53,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 7404e7668f405a8889810821ba8518dd in 898ms, sequenceid=208, compaction requested=true 2024-12-08T00:15:53,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:53,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:53,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:53,782 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:53,782 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:53,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:53,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:53,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:53,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:53,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:53,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/A is initiating minor compaction (all files) 2024-12-08T00:15:53,787 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/A in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:53,787 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/1ce05430ffae4839b715cc31a690ab9a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f0f8ed576f6e41daa797001dfefecb73, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/13df0183b38145879c92a34f5223796d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=100.0 K 2024-12-08T00:15:53,787 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:53,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/1ce05430ffae4839b715cc31a690ab9a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f0f8ed576f6e41daa797001dfefecb73, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/13df0183b38145879c92a34f5223796d] 2024-12-08T00:15:53,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:53,788 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/B is initiating minor compaction (all files) 2024-12-08T00:15:53,788 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/B in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:53,788 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/23719d7d142447ceba090650cfe6db67, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/8d9038f5767f451fab8abffd77420cbe, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/c21206e878f64937aab1680587f00ece] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=36.0 K 2024-12-08T00:15:53,788 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 23719d7d142447ceba090650cfe6db67, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733616949824 2024-12-08T00:15:53,788 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce05430ffae4839b715cc31a690ab9a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733616949824 2024-12-08T00:15:53,791 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f8ed576f6e41daa797001dfefecb73, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733616951002 2024-12-08T00:15:53,792 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d9038f5767f451fab8abffd77420cbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733616951002 2024-12-08T00:15:53,792 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13df0183b38145879c92a34f5223796d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616952226 2024-12-08T00:15:53,792 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c21206e878f64937aab1680587f00ece, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616952226 2024-12-08T00:15:53,813 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:53,823 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#B#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:53,823 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/3210307c7dcc4efb942fce65dbedc811 is 50, key is test_row_0/B:col10/1733616952882/Put/seqid=0 2024-12-08T00:15:53,828 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208863553a6cc18492fbf07284cff350c92_7404e7668f405a8889810821ba8518dd store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:53,830 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208863553a6cc18492fbf07284cff350c92_7404e7668f405a8889810821ba8518dd, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:53,831 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208863553a6cc18492fbf07284cff350c92_7404e7668f405a8889810821ba8518dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:53,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742175_1351 (size=12663) 2024-12-08T00:15:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T00:15:53,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742176_1352 (size=4469) 2024-12-08T00:15:53,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T00:15:53,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:53,899 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T00:15:53,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:53,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:53,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:53,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:53,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:53,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:53,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120826ca9c00d883488a888d7239b1913354_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616952926/Put/seqid=0 2024-12-08T00:15:53,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742177_1353 (size=12304) 2024-12-08T00:15:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T00:15:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:54,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:54,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617014129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617014132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,252 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/3210307c7dcc4efb942fce65dbedc811 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3210307c7dcc4efb942fce65dbedc811 2024-12-08T00:15:54,258 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#A#compaction#291 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:54,259 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/708050dca22d49f4834936a9fd1e5010 is 175, key is test_row_0/A:col10/1733616952882/Put/seqid=0 2024-12-08T00:15:54,260 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/B of 7404e7668f405a8889810821ba8518dd into 3210307c7dcc4efb942fce65dbedc811(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:54,260 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:54,260 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/B, priority=13, startTime=1733616953782; duration=0sec 2024-12-08T00:15:54,260 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:54,260 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:B 2024-12-08T00:15:54,260 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:54,261 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:54,261 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/C is initiating minor compaction (all files) 2024-12-08T00:15:54,261 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/C in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:54,261 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/fd99674c2b5b42cd88a2d4ea57f4bde7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e13b8fdacc224d21a348ab7e43f5b6b3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3b698d86a71046fe84bd969253271289] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=36.0 K 2024-12-08T00:15:54,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617014251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,262 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting fd99674c2b5b42cd88a2d4ea57f4bde7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733616949824 2024-12-08T00:15:54,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,262 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e13b8fdacc224d21a348ab7e43f5b6b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733616951002 2024-12-08T00:15:54,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,263 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b698d86a71046fe84bd969253271289, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616952226 2024-12-08T00:15:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617014252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742178_1354 (size=31617) 2024-12-08T00:15:54,280 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/708050dca22d49f4834936a9fd1e5010 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/708050dca22d49f4834936a9fd1e5010 2024-12-08T00:15:54,284 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:54,285 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/A of 7404e7668f405a8889810821ba8518dd into 708050dca22d49f4834936a9fd1e5010(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:54,285 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:54,285 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/83025f3585654cde88b1df702edbdadc is 50, key is test_row_0/C:col10/1733616952882/Put/seqid=0 2024-12-08T00:15:54,285 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/A, priority=13, startTime=1733616953782; duration=0sec 2024-12-08T00:15:54,285 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:54,285 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:A 2024-12-08T00:15:54,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742179_1355 (size=12663) 2024-12-08T00:15:54,293 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/83025f3585654cde88b1df702edbdadc as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/83025f3585654cde88b1df702edbdadc 2024-12-08T00:15:54,298 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/C of 7404e7668f405a8889810821ba8518dd into 83025f3585654cde88b1df702edbdadc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:54,298 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:54,298 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/C, priority=13, startTime=1733616953782; duration=0sec 2024-12-08T00:15:54,299 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:54,299 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:C 2024-12-08T00:15:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:54,314 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120826ca9c00d883488a888d7239b1913354_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120826ca9c00d883488a888d7239b1913354_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fec90fc54d9442809a2adcf7e555a25f, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fec90fc54d9442809a2adcf7e555a25f is 175, key is test_row_0/A:col10/1733616952926/Put/seqid=0 2024-12-08T00:15:54,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742180_1356 (size=31105) 2024-12-08T00:15:54,326 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fec90fc54d9442809a2adcf7e555a25f 2024-12-08T00:15:54,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/415d9ee246ee45e08b6a6e9ce7b19483 is 50, key is test_row_0/B:col10/1733616952926/Put/seqid=0 2024-12-08T00:15:54,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742181_1357 (size=12151) 2024-12-08T00:15:54,335 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/415d9ee246ee45e08b6a6e9ce7b19483 2024-12-08T00:15:54,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/bccbeb3f5c5946f786a5c002c7571b40 is 50, key is test_row_0/C:col10/1733616952926/Put/seqid=0 2024-12-08T00:15:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742182_1358 (size=12151) 2024-12-08T00:15:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T00:15:54,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617014470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617014470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,744 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/bccbeb3f5c5946f786a5c002c7571b40 2024-12-08T00:15:54,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fec90fc54d9442809a2adcf7e555a25f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fec90fc54d9442809a2adcf7e555a25f 2024-12-08T00:15:54,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fec90fc54d9442809a2adcf7e555a25f, entries=150, sequenceid=224, filesize=30.4 K 2024-12-08T00:15:54,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/415d9ee246ee45e08b6a6e9ce7b19483 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/415d9ee246ee45e08b6a6e9ce7b19483 2024-12-08T00:15:54,766 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/415d9ee246ee45e08b6a6e9ce7b19483, entries=150, sequenceid=224, filesize=11.9 K 2024-12-08T00:15:54,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/bccbeb3f5c5946f786a5c002c7571b40 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/bccbeb3f5c5946f786a5c002c7571b40 2024-12-08T00:15:54,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/bccbeb3f5c5946f786a5c002c7571b40, entries=150, sequenceid=224, filesize=11.9 K 2024-12-08T00:15:54,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7404e7668f405a8889810821ba8518dd in 880ms, sequenceid=224, compaction requested=false 2024-12-08T00:15:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-08T00:15:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-08T00:15:54,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-08T00:15:54,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0360 sec 2024-12-08T00:15:54,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.0400 sec 2024-12-08T00:15:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:54,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T00:15:54,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:54,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:54,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:54,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:54,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:54,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:54,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ba2efa8a176a4d3abd737dcfd6e48329_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616954130/Put/seqid=0 2024-12-08T00:15:54,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742183_1359 (size=12304) 2024-12-08T00:15:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617014842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617014849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T00:15:54,881 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:54,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-08T00:15:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T00:15:54,883 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:54,883 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:54,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:54,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617014950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617014961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T00:15:55,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T00:15:55,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:55,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617015160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:55,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617015170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:55,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T00:15:55,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:55,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T00:15:55,205 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:55,208 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ba2efa8a176a4d3abd737dcfd6e48329_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ba2efa8a176a4d3abd737dcfd6e48329_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:55,209 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/bb3dc368c3c641d08073a448b7075f59, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:55,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/bb3dc368c3c641d08073a448b7075f59 is 175, key is test_row_0/A:col10/1733616954130/Put/seqid=0 2024-12-08T00:15:55,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742184_1360 (size=31105) 2024-12-08T00:15:55,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T00:15:55,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:55,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617015471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:55,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617015480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:55,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T00:15:55,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:55,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T00:15:55,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/bb3dc368c3c641d08073a448b7075f59 2024-12-08T00:15:55,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/1c9a12f04ab9450d968b3c8eee7203a4 is 50, key is test_row_0/B:col10/1733616954130/Put/seqid=0 2024-12-08T00:15:55,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742185_1361 (size=12151) 2024-12-08T00:15:55,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T00:15:55,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:55,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T00:15:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T00:15:55,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:55,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:55,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:55,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617015981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:56,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617016001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:56,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:56,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:56,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T00:15:56,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/1c9a12f04ab9450d968b3c8eee7203a4 2024-12-08T00:15:56,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/ee614ee6b00d461c991eebb2dca25afb is 50, key is test_row_0/C:col10/1733616954130/Put/seqid=0 2024-12-08T00:15:56,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742186_1362 (size=12151) 2024-12-08T00:15:56,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/ee614ee6b00d461c991eebb2dca25afb 2024-12-08T00:15:56,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/bb3dc368c3c641d08073a448b7075f59 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/bb3dc368c3c641d08073a448b7075f59 2024-12-08T00:15:56,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/bb3dc368c3c641d08073a448b7075f59, entries=150, sequenceid=244, filesize=30.4 K 2024-12-08T00:15:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/1c9a12f04ab9450d968b3c8eee7203a4 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/1c9a12f04ab9450d968b3c8eee7203a4 2024-12-08T00:15:56,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/1c9a12f04ab9450d968b3c8eee7203a4, entries=150, sequenceid=244, filesize=11.9 K 2024-12-08T00:15:56,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/ee614ee6b00d461c991eebb2dca25afb as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/ee614ee6b00d461c991eebb2dca25afb 2024-12-08T00:15:56,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/ee614ee6b00d461c991eebb2dca25afb, entries=150, sequenceid=244, filesize=11.9 K 2024-12-08T00:15:56,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 7404e7668f405a8889810821ba8518dd in 1278ms, sequenceid=244, compaction requested=true 2024-12-08T00:15:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:56,066 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:56,066 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:56,067 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:56,067 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/A is initiating minor compaction (all files) 2024-12-08T00:15:56,067 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/A in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:56,067 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/708050dca22d49f4834936a9fd1e5010, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fec90fc54d9442809a2adcf7e555a25f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/bb3dc368c3c641d08073a448b7075f59] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=91.6 K 2024-12-08T00:15:56,067 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:56,067 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/708050dca22d49f4834936a9fd1e5010, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fec90fc54d9442809a2adcf7e555a25f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/bb3dc368c3c641d08073a448b7075f59] 2024-12-08T00:15:56,067 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:56,067 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/B is initiating minor compaction (all files) 2024-12-08T00:15:56,067 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/B in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:56,067 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708050dca22d49f4834936a9fd1e5010, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616952226 2024-12-08T00:15:56,067 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3210307c7dcc4efb942fce65dbedc811, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/415d9ee246ee45e08b6a6e9ce7b19483, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/1c9a12f04ab9450d968b3c8eee7203a4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=36.1 K 2024-12-08T00:15:56,068 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting fec90fc54d9442809a2adcf7e555a25f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733616952900 2024-12-08T00:15:56,068 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 3210307c7dcc4efb942fce65dbedc811, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616952226 2024-12-08T00:15:56,068 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb3dc368c3c641d08073a448b7075f59, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733616954112 2024-12-08T00:15:56,068 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 415d9ee246ee45e08b6a6e9ce7b19483, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733616952900 2024-12-08T00:15:56,068 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c9a12f04ab9450d968b3c8eee7203a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733616954112 2024-12-08T00:15:56,074 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:56,076 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:56,076 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/2952c8ef7ace4de7929f90a2d42c4f0a is 50, key is test_row_0/B:col10/1733616954130/Put/seqid=0 2024-12-08T00:15:56,077 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412081f9dc21bfb1e4d92aef3c3abadbcd4cc_7404e7668f405a8889810821ba8518dd store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:56,078 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412081f9dc21bfb1e4d92aef3c3abadbcd4cc_7404e7668f405a8889810821ba8518dd, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:56,079 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081f9dc21bfb1e4d92aef3c3abadbcd4cc_7404e7668f405a8889810821ba8518dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:56,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742187_1363 (size=12765) 2024-12-08T00:15:56,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742188_1364 (size=4469) 2024-12-08T00:15:56,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T00:15:56,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:56,103 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T00:15:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:56,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208030ec80ec97942e59b303a6be298fec8_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616954847/Put/seqid=0 2024-12-08T00:15:56,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742189_1365 (size=12404) 2024-12-08T00:15:56,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,117 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208030ec80ec97942e59b303a6be298fec8_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208030ec80ec97942e59b303a6be298fec8_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:56,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/d76db06456d64180a17edc6e463d7c89, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:56,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/d76db06456d64180a17edc6e463d7c89 is 175, key is test_row_0/A:col10/1733616954847/Put/seqid=0 2024-12-08T00:15:56,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742190_1366 (size=31205) 2024-12-08T00:15:56,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/d76db06456d64180a17edc6e463d7c89 2024-12-08T00:15:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/d1787433fa14476388b4e055d3412953 is 50, key is test_row_0/B:col10/1733616954847/Put/seqid=0 2024-12-08T00:15:56,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742191_1367 (size=12251) 2024-12-08T00:15:56,483 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#A#compaction#301 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:56,483 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6111f5c366ef4afcbaeb7b5497ab534c is 175, key is test_row_0/A:col10/1733616954130/Put/seqid=0 2024-12-08T00:15:56,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742192_1368 (size=31719) 2024-12-08T00:15:56,487 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/2952c8ef7ace4de7929f90a2d42c4f0a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/2952c8ef7ace4de7929f90a2d42c4f0a 2024-12-08T00:15:56,490 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6111f5c366ef4afcbaeb7b5497ab534c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6111f5c366ef4afcbaeb7b5497ab534c 2024-12-08T00:15:56,491 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/B of 7404e7668f405a8889810821ba8518dd into 2952c8ef7ace4de7929f90a2d42c4f0a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:56,491 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:56,491 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/B, priority=13, startTime=1733616956066; duration=0sec 2024-12-08T00:15:56,491 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:56,491 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:B 2024-12-08T00:15:56,491 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:56,492 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:56,492 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/C is initiating minor compaction (all files) 2024-12-08T00:15:56,492 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/C in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:56,492 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/83025f3585654cde88b1df702edbdadc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/bccbeb3f5c5946f786a5c002c7571b40, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/ee614ee6b00d461c991eebb2dca25afb] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=36.1 K 2024-12-08T00:15:56,492 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 83025f3585654cde88b1df702edbdadc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733616952226 2024-12-08T00:15:56,492 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting bccbeb3f5c5946f786a5c002c7571b40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733616952900 2024-12-08T00:15:56,493 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ee614ee6b00d461c991eebb2dca25afb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733616954112 2024-12-08T00:15:56,494 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/A of 7404e7668f405a8889810821ba8518dd into 6111f5c366ef4afcbaeb7b5497ab534c(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:56,494 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:56,494 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/A, priority=13, startTime=1733616956066; duration=0sec 2024-12-08T00:15:56,495 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:56,495 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:A 2024-12-08T00:15:56,498 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#C#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:56,499 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/9bced484a44d465d880ccfc21a2b1515 is 50, key is test_row_0/C:col10/1733616954130/Put/seqid=0 2024-12-08T00:15:56,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742193_1369 (size=12765) 2024-12-08T00:15:56,549 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/d1787433fa14476388b4e055d3412953 2024-12-08T00:15:56,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/3124b4b595894bd19078d18dd2d1887b is 50, key is test_row_0/C:col10/1733616954847/Put/seqid=0 2024-12-08T00:15:56,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742194_1370 (size=12251) 2024-12-08T00:15:56,906 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/9bced484a44d465d880ccfc21a2b1515 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9bced484a44d465d880ccfc21a2b1515 2024-12-08T00:15:56,915 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/C of 7404e7668f405a8889810821ba8518dd into 9bced484a44d465d880ccfc21a2b1515(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:56,915 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:56,915 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/C, priority=13, startTime=1733616956066; duration=0sec 2024-12-08T00:15:56,916 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:56,916 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:C 2024-12-08T00:15:56,959 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/3124b4b595894bd19078d18dd2d1887b 2024-12-08T00:15:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/d76db06456d64180a17edc6e463d7c89 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d76db06456d64180a17edc6e463d7c89 2024-12-08T00:15:56,967 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d76db06456d64180a17edc6e463d7c89, entries=150, sequenceid=259, filesize=30.5 K 2024-12-08T00:15:56,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/d1787433fa14476388b4e055d3412953 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/d1787433fa14476388b4e055d3412953 2024-12-08T00:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,972 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/d1787433fa14476388b4e055d3412953, entries=150, sequenceid=259, filesize=12.0 K 2024-12-08T00:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/3124b4b595894bd19078d18dd2d1887b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3124b4b595894bd19078d18dd2d1887b 2024-12-08T00:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3124b4b595894bd19078d18dd2d1887b, entries=150, sequenceid=259, filesize=12.0 K 2024-12-08T00:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,977 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 7404e7668f405a8889810821ba8518dd in 874ms, sequenceid=259, compaction requested=false 2024-12-08T00:15:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-08T00:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-08T00:15:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-08T00:15:56,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0950 sec 2024-12-08T00:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.1000 sec 2024-12-08T00:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T00:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,031 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-08T00:15:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T00:15:57,033 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:15:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,034 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T00:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:57,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:15:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:15:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ae18297ea7154f89a40087b1fb2c0b98_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616957146/Put/seqid=0 2024-12-08T00:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:57,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742195_1371 (size=12454) 2024-12-08T00:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,192 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,195 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ae18297ea7154f89a40087b1fb2c0b98_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ae18297ea7154f89a40087b1fb2c0b98_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:57,196 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6de7b917c69b4d81a3e74dde0b800d34, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:57,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6de7b917c69b4d81a3e74dde0b800d34 is 175, key is test_row_0/A:col10/1733616957146/Put/seqid=0 2024-12-08T00:15:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742196_1372 (size=31251) 2024-12-08T00:15:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,206 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6de7b917c69b4d81a3e74dde0b800d34 2024-12-08T00:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:57,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/ed56b56c562542709c1b70ab389e6b09 is 50, key is test_row_0/B:col10/1733616957146/Put/seqid=0 2024-12-08T00:15:57,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742197_1373 (size=9857) 2024-12-08T00:15:57,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/ed56b56c562542709c1b70ab389e6b09 2024-12-08T00:15:57,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/9f8cc70efac24d128faf1e15455d315b is 50, key is test_row_0/C:col10/1733616957146/Put/seqid=0 2024-12-08T00:15:57,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742198_1374 (size=9857) 2024-12-08T00:15:57,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616957235, max timestamp=1733616957273Num uniques 859; 2024-12-08T00:15:57,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:15:57,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616957235, max timestamp=1733616957272Num uniques 859; 2024-12-08T00:15:57,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616957235, max timestamp=1733616957273Num uniques 859; 2024-12-08T00:15:57,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:15:57,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:15:57,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T00:15:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617017350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:57,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617017357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:57,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:57,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:57,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617017461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:57,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617017471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:57,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:57,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:57,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:57,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/9f8cc70efac24d128faf1e15455d315b 2024-12-08T00:15:57,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:57,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:57,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6de7b917c69b4d81a3e74dde0b800d34 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6de7b917c69b4d81a3e74dde0b800d34 2024-12-08T00:15:57,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6de7b917c69b4d81a3e74dde0b800d34, entries=150, sequenceid=272, filesize=30.5 K 2024-12-08T00:15:57,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/ed56b56c562542709c1b70ab389e6b09 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ed56b56c562542709c1b70ab389e6b09 2024-12-08T00:15:57,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ed56b56c562542709c1b70ab389e6b09, entries=100, sequenceid=272, filesize=9.6 K 2024-12-08T00:15:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/9f8cc70efac24d128faf1e15455d315b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9f8cc70efac24d128faf1e15455d315b 2024-12-08T00:15:57,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9f8cc70efac24d128faf1e15455d315b, entries=100, sequenceid=272, filesize=9.6 K 2024-12-08T00:15:57,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7404e7668f405a8889810821ba8518dd in 504ms, sequenceid=272, compaction requested=true 2024-12-08T00:15:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T00:15:57,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:57,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:15:57,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:57,662 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:57,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:15:57,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:57,662 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:57,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:15:57,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:57,662 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:57,662 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/A is initiating minor compaction (all files) 2024-12-08T00:15:57,662 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/A in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,662 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6111f5c366ef4afcbaeb7b5497ab534c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d76db06456d64180a17edc6e463d7c89, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6de7b917c69b4d81a3e74dde0b800d34] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=92.0 K 2024-12-08T00:15:57,663 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,663 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6111f5c366ef4afcbaeb7b5497ab534c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d76db06456d64180a17edc6e463d7c89, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6de7b917c69b4d81a3e74dde0b800d34] 2024-12-08T00:15:57,663 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:57,663 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/B is initiating minor compaction (all files) 2024-12-08T00:15:57,663 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/B in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,663 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6111f5c366ef4afcbaeb7b5497ab534c, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733616954112 2024-12-08T00:15:57,663 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/2952c8ef7ace4de7929f90a2d42c4f0a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/d1787433fa14476388b4e055d3412953, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ed56b56c562542709c1b70ab389e6b09] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=34.1 K 2024-12-08T00:15:57,663 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting d76db06456d64180a17edc6e463d7c89, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733616954836 2024-12-08T00:15:57,663 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 2952c8ef7ace4de7929f90a2d42c4f0a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733616954112 2024-12-08T00:15:57,663 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6de7b917c69b4d81a3e74dde0b800d34, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616957105 2024-12-08T00:15:57,663 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d1787433fa14476388b4e055d3412953, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733616954836 2024-12-08T00:15:57,664 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ed56b56c562542709c1b70ab389e6b09, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616957129 2024-12-08T00:15:57,669 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:57,669 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:57,669 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/75e046df12704dc19d8017bc28320001 is 50, key is test_row_0/B:col10/1733616957146/Put/seqid=0 2024-12-08T00:15:57,672 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208875d7ba9c128436d9c7a9e16375595dc_7404e7668f405a8889810821ba8518dd store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:57,673 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208875d7ba9c128436d9c7a9e16375595dc_7404e7668f405a8889810821ba8518dd, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:57,673 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208875d7ba9c128436d9c7a9e16375595dc_7404e7668f405a8889810821ba8518dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:57,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733616957272, max timestamp=1733616957676Num uniques 1018; 2024-12-08T00:15:57,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T00:15:57,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733616957272, max timestamp=1733616957676Num uniques 1018; 2024-12-08T00:15:57,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T00:15:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:57,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733616957272, max timestamp=1733616957676Num uniques 1018; 2024-12-08T00:15:57,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T00:15:57,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-12-08T00:15:57,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:57,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:57,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:57,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:57,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:57,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:57,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742199_1375 (size=13017) 2024-12-08T00:15:57,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742200_1376 (size=4469) 2024-12-08T00:15:57,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208778ee66bf07a4a1fb51d89b6b865e5c3_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616957676/Put/seqid=0 2024-12-08T00:15:57,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742201_1377 (size=14994) 2024-12-08T00:15:57,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616957726, max timestamp=1733616957737Num uniques 859; 2024-12-08T00:15:57,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:15:57,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616957726, max timestamp=1733616957737Num uniques 859; 2024-12-08T00:15:57,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:15:57,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616957726, max timestamp=1733616957737Num uniques 859; 2024-12-08T00:15:57,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:15:57,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733616957766, max timestamp=1733616957803Num uniques 1068; 2024-12-08T00:15:57,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-08T00:15:57,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733616957766, max timestamp=1733616957803Num uniques 1068; 2024-12-08T00:15:57,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733616957766, max timestamp=1733616957803Num uniques 1068; 2024-12-08T00:15:57,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T00:15:57,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T00:15:57,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733616957838, max timestamp=1733616957884Num uniques 1027; 2024-12-08T00:15:57,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733616957838, max timestamp=1733616957884Num uniques 1127; 2024-12-08T00:15:57,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-08T00:15:57,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T00:15:57,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733616957838, max timestamp=1733616957884Num uniques 1027; 2024-12-08T00:15:57,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T00:15:57,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:57,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:57,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733616957950, max timestamp=1733616957965Num uniques 1086; 2024-12-08T00:15:57,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-08T00:15:57,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733616957950, max timestamp=1733616957965Num uniques 1086; 2024-12-08T00:15:57,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733616957950, max timestamp=1733616957965Num uniques 1086; 2024-12-08T00:15:57,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-08T00:15:57,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-08T00:15:58,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733616958002, max timestamp=1733616958042Num uniques 1095; 2024-12-08T00:15:58,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-08T00:15:58,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733616958002, max timestamp=1733616958042Num uniques 1095; 2024-12-08T00:15:58,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-08T00:15:58,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733616958002, max timestamp=1733616958042Num uniques 1095; 2024-12-08T00:15:58,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-08T00:15:58,087 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:58,088 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fd2668d514ec4c2bbf591dbcd55fc25d is 175, key is test_row_0/A:col10/1733616957146/Put/seqid=0 2024-12-08T00:15:58,088 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/75e046df12704dc19d8017bc28320001 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/75e046df12704dc19d8017bc28320001 2024-12-08T00:15:58,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742202_1378 (size=32078) 2024-12-08T00:15:58,092 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/B of 7404e7668f405a8889810821ba8518dd into 75e046df12704dc19d8017bc28320001(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:58,092 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:58,092 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/B, priority=13, startTime=1733616957662; duration=0sec 2024-12-08T00:15:58,093 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:15:58,093 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:B 2024-12-08T00:15:58,093 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:15:58,093 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:58,096 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:15:58,096 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/C is initiating minor compaction (all files) 2024-12-08T00:15:58,096 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/C in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,096 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9bced484a44d465d880ccfc21a2b1515, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3124b4b595894bd19078d18dd2d1887b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9f8cc70efac24d128faf1e15455d315b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=34.1 K 2024-12-08T00:15:58,096 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fd2668d514ec4c2bbf591dbcd55fc25d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fd2668d514ec4c2bbf591dbcd55fc25d 2024-12-08T00:15:58,097 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bced484a44d465d880ccfc21a2b1515, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733616954112 2024-12-08T00:15:58,097 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 3124b4b595894bd19078d18dd2d1887b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733616954836 2024-12-08T00:15:58,097 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f8cc70efac24d128faf1e15455d315b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616957129 2024-12-08T00:15:58,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208778ee66bf07a4a1fb51d89b6b865e5c3_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208778ee66bf07a4a1fb51d89b6b865e5c3_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:58,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/463ab095e1dd4cc09578fa21ab54ed7b, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:58,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/463ab095e1dd4cc09578fa21ab54ed7b is 175, key is test_row_0/A:col10/1733616957676/Put/seqid=0 2024-12-08T00:15:58,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:58,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:58,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,103 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/A of 7404e7668f405a8889810821ba8518dd into fd2668d514ec4c2bbf591dbcd55fc25d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:58,103 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:58,103 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/A, priority=13, startTime=1733616957661; duration=0sec 2024-12-08T00:15:58,104 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:58,104 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:A 2024-12-08T00:15:58,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742203_1379 (size=39949) 2024-12-08T00:15:58,116 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:15:58,117 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/19c62ad56d334cfb9a173bb4f176aecb is 50, key is test_row_0/C:col10/1733616957146/Put/seqid=0 2024-12-08T00:15:58,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742204_1380 (size=13017) 2024-12-08T00:15:58,125 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/19c62ad56d334cfb9a173bb4f176aecb as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/19c62ad56d334cfb9a173bb4f176aecb 2024-12-08T00:15:58,129 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/C of 7404e7668f405a8889810821ba8518dd into 19c62ad56d334cfb9a173bb4f176aecb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:15:58,129 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:58,129 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/C, priority=13, startTime=1733616957662; duration=0sec 2024-12-08T00:15:58,129 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:15:58,129 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:C 2024-12-08T00:15:58,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733616958116, max timestamp=1733616958138Num uniques 1154; 2024-12-08T00:15:58,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-12-08T00:15:58,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733616958116, max timestamp=1733616958138Num uniques 1154; 2024-12-08T00:15:58,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-12-08T00:15:58,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733616958116, max timestamp=1733616958138Num uniques 1154; 2024-12-08T00:15:58,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-12-08T00:15:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T00:15:58,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733616958176, max timestamp=1733616958205Num uniques 1063; 2024-12-08T00:15:58,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-08T00:15:58,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733616958176, max timestamp=1733616958205Num uniques 1063; 2024-12-08T00:15:58,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-08T00:15:58,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733616958176, max timestamp=1733616958205Num uniques 1063; 2024-12-08T00:15:58,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-08T00:15:58,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:58,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:58,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733616958230, max timestamp=1733616958281Num uniques 1022; 2024-12-08T00:15:58,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-08T00:15:58,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733616958230, max timestamp=1733616958281Num uniques 1022; 2024-12-08T00:15:58,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-08T00:15:58,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733616958230, max timestamp=1733616958281Num uniques 1022; 2024-12-08T00:15:58,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-08T00:15:58,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733616958339, max timestamp=1733616958361Num uniques 1081; 2024-12-08T00:15:58,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733616958339, max timestamp=1733616958361Num uniques 1081; 2024-12-08T00:15:58,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-08T00:15:58,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-08T00:15:58,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733616958339, max timestamp=1733616958361Num uniques 1081; 2024-12-08T00:15:58,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-08T00:15:58,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:58,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733616958390, max timestamp=1733616958417Num uniques 1040; 2024-12-08T00:15:58,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-08T00:15:58,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733616958390, max timestamp=1733616958417Num uniques 1040; 2024-12-08T00:15:58,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-08T00:15:58,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733616958390, max timestamp=1733616958417Num uniques 1040; 2024-12-08T00:15:58,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-08T00:15:58,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733616958483, max timestamp=1733616958501Num uniques 1049; 2024-12-08T00:15:58,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-12-08T00:15:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733616958483, max timestamp=1733616958501Num uniques 1049; 2024-12-08T00:15:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733616958483, max timestamp=1733616958501Num uniques 1049; 2024-12-08T00:15:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-12-08T00:15:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-12-08T00:15:58,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/463ab095e1dd4cc09578fa21ab54ed7b 2024-12-08T00:15:58,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/dbb32111b28c4487bec3051cabd04d5f is 50, key is test_row_0/B:col10/1733616957676/Put/seqid=0 2024-12-08T00:15:58,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742205_1381 (size=12301) 2024-12-08T00:15:58,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617018561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733616958557, max timestamp=1733616958570Num uniques 1158; 2024-12-08T00:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-12-08T00:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733616958557, max timestamp=1733616958570Num uniques 1158; 2024-12-08T00:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-12-08T00:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733616958557, max timestamp=1733616958570Num uniques 1158; 2024-12-08T00:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-12-08T00:15:58,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733616958610, max timestamp=1733616958655Num uniques 1017; 2024-12-08T00:15:58,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-08T00:15:58,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733616958610, max timestamp=1733616958655Num uniques 1017; 2024-12-08T00:15:58,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-08T00:15:58,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733616958610, max timestamp=1733616958655Num uniques 1017; 2024-12-08T00:15:58,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-08T00:15:58,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733616958685, max timestamp=1733616958729Num uniques 1076; 2024-12-08T00:15:58,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-08T00:15:58,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733616958685, max timestamp=1733616958729Num uniques 1076; 2024-12-08T00:15:58,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733616958685, max timestamp=1733616958729Num uniques 1076; 2024-12-08T00:15:58,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-08T00:15:58,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-08T00:15:58,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733616958777, max timestamp=1733616958788Num uniques 1085; 2024-12-08T00:15:58,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733616958777, max timestamp=1733616958788Num uniques 1085; 2024-12-08T00:15:58,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-08T00:15:58,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-08T00:15:58,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733616958777, max timestamp=1733616958788Num uniques 1085; 2024-12-08T00:15:58,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-08T00:15:58,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733616958836, max timestamp=1733616958853Num uniques 1044; 2024-12-08T00:15:58,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-08T00:15:58,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733616958836, max timestamp=1733616958853Num uniques 1044; 2024-12-08T00:15:58,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-08T00:15:58,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733616958836, max timestamp=1733616958853Num uniques 1044; 2024-12-08T00:15:58,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-08T00:15:58,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:58,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:58,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733616958896, max timestamp=1733616958917Num uniques 1053; 2024-12-08T00:15:58,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733616958896, max timestamp=1733616958917Num uniques 1053; 2024-12-08T00:15:58,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-12-08T00:15:58,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-12-08T00:15:58,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733616958896, max timestamp=1733616958917Num uniques 1053; 2024-12-08T00:15:58,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-12-08T00:15:58,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/dbb32111b28c4487bec3051cabd04d5f 2024-12-08T00:15:58,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/143357098f144e7ea7933fda10add313 is 50, key is test_row_0/C:col10/1733616957676/Put/seqid=0 2024-12-08T00:15:58,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742206_1382 (size=12301) 2024-12-08T00:15:58,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733616958948, max timestamp=1733616958981Num uniques 1062; 2024-12-08T00:15:58,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733616958948, max timestamp=1733616958981Num uniques 1062; 2024-12-08T00:15:58,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-12-08T00:15:58,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-12-08T00:15:58,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733616958948, max timestamp=1733616958981Num uniques 1062; 2024-12-08T00:15:58,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-12-08T00:15:59,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:59,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:59,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733616959031, max timestamp=1733616959051Num uniques 1071; 2024-12-08T00:15:59,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=7826, suffix heap size=139352,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=171 2024-12-08T00:15:59,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733616959031, max timestamp=1733616959051Num uniques 1071; 2024-12-08T00:15:59,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=7826, suffix heap size=139352,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=171 2024-12-08T00:15:59,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733616959031, max timestamp=1733616959051Num uniques 1071; 2024-12-08T00:15:59,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=7826, suffix heap size=139352,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=171 2024-12-08T00:15:59,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1733616959098, max timestamp=1733616959119Num uniques 1030; 2024-12-08T00:15:59,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-12-08T00:15:59,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1733616959098, max timestamp=1733616959119Num uniques 1030; 2024-12-08T00:15:59,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-12-08T00:15:59,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1733616959098, max timestamp=1733616959119Num uniques 1030; 2024-12-08T00:15:59,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-12-08T00:15:59,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1733616959158, max timestamp=1733616959177Num uniques 1039; 2024-12-08T00:15:59,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=10944, suffix heap size=136024,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=239 2024-12-08T00:15:59,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1733616959158, max timestamp=1733616959177Num uniques 1039; 2024-12-08T00:15:59,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=10944, suffix heap size=136024,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=239 2024-12-08T00:15:59,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1733616959158, max timestamp=1733616959177Num uniques 1039; 2024-12-08T00:15:59,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=10944, suffix heap size=136024,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=239 2024-12-08T00:15:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T00:15:59,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1733616959213, max timestamp=1733616959236Num uniques 1098; 2024-12-08T00:15:59,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=9068, suffix heap size=142160,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=198 2024-12-08T00:15:59,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1733616959213, max timestamp=1733616959236Num uniques 1098; 2024-12-08T00:15:59,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=9068, suffix heap size=142160,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=198 2024-12-08T00:15:59,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1733616959213, max timestamp=1733616959236Num uniques 1098; 2024-12-08T00:15:59,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=9068, suffix heap size=142160,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=198 2024-12-08T00:15:59,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1733616959272, max timestamp=1733616959289Num uniques 1057; 2024-12-08T00:15:59,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=9475, suffix heap size=137896,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=207 2024-12-08T00:15:59,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1733616959272, max timestamp=1733616959289Num uniques 1057; 2024-12-08T00:15:59,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=9475, suffix heap size=137896,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=207 2024-12-08T00:15:59,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1733616959272, max timestamp=1733616959289Num uniques 1057; 2024-12-08T00:15:59,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=9475, suffix heap size=137896,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=207 2024-12-08T00:15:59,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:59,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:59,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733616959313, max timestamp=1733616959338Num uniques 1066; 2024-12-08T00:15:59,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733616959313, max timestamp=1733616959338Num uniques 1066; 2024-12-08T00:15:59,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=9886, suffix heap size=138832,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=216 2024-12-08T00:15:59,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=9886, suffix heap size=138832,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=216 2024-12-08T00:15:59,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733616959313, max timestamp=1733616959338Num uniques 1066; 2024-12-08T00:15:59,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=9886, suffix heap size=138832,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=216 2024-12-08T00:15:59,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/143357098f144e7ea7933fda10add313 2024-12-08T00:15:59,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/463ab095e1dd4cc09578fa21ab54ed7b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/463ab095e1dd4cc09578fa21ab54ed7b 2024-12-08T00:15:59,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/463ab095e1dd4cc09578fa21ab54ed7b, entries=200, sequenceid=310, filesize=39.0 K 2024-12-08T00:15:59,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/dbb32111b28c4487bec3051cabd04d5f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/dbb32111b28c4487bec3051cabd04d5f 2024-12-08T00:15:59,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/dbb32111b28c4487bec3051cabd04d5f, entries=150, sequenceid=310, filesize=12.0 K 2024-12-08T00:15:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/143357098f144e7ea7933fda10add313 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/143357098f144e7ea7933fda10add313 2024-12-08T00:15:59,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/143357098f144e7ea7933fda10add313, entries=150, sequenceid=310, filesize=12.0 K 2024-12-08T00:15:59,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=93.93 KB/96180 for 7404e7668f405a8889810821ba8518dd in 1684ms, sequenceid=310, compaction requested=false 2024-12-08T00:15:59,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:15:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:59,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=100.63 KB heapSize=280.03 KB 2024-12-08T00:15:59,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:15:59,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:59,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:15:59,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:59,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:15:59,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:15:59,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c2b32599258c40bda19e3d8acc6c4b37_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616959365/Put/seqid=0 2024-12-08T00:15:59,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742207_1383 (size=14994) 2024-12-08T00:15:59,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 376 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617019386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617019395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617019396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617019396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:59,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:59,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 377 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617019501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617019510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617019511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617019511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:59,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 378 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617019720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617019730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 355 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617019730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617019731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:15:59,777 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:15:59,781 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c2b32599258c40bda19e3d8acc6c4b37_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c2b32599258c40bda19e3d8acc6c4b37_7404e7668f405a8889810821ba8518dd 2024-12-08T00:15:59,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/09e79c45fa024767a31f92e2cd0e4f16, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:15:59,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/09e79c45fa024767a31f92e2cd0e4f16 is 175, key is test_row_0/A:col10/1733616959365/Put/seqid=0 2024-12-08T00:15:59,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:59,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:59,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742208_1384 (size=39949) 2024-12-08T00:15:59,791 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=739, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/09e79c45fa024767a31f92e2cd0e4f16 2024-12-08T00:15:59,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/f13e0f73a45f43e09a88b979b29e0f23 is 50, key is test_row_0/B:col10/1733616959365/Put/seqid=0 2024-12-08T00:15:59,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742209_1385 (size=12301) 2024-12-08T00:15:59,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:15:59,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:15:59,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:15:59,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:15:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:00,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 379 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617020031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617020051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 356 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617020051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617020051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:16:00,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:00,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:16:00,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:00,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:00,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:00,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=739 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/f13e0f73a45f43e09a88b979b29e0f23 2024-12-08T00:16:00,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/b73662665c864dc58bc17f27aac282ca is 50, key is test_row_0/C:col10/1733616959365/Put/seqid=0 2024-12-08T00:16:00,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742210_1386 (size=12301) 2024-12-08T00:16:00,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=739 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/b73662665c864dc58bc17f27aac282ca 2024-12-08T00:16:00,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/09e79c45fa024767a31f92e2cd0e4f16 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/09e79c45fa024767a31f92e2cd0e4f16 2024-12-08T00:16:00,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/09e79c45fa024767a31f92e2cd0e4f16, entries=200, sequenceid=739, filesize=39.0 K 2024-12-08T00:16:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/f13e0f73a45f43e09a88b979b29e0f23 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/f13e0f73a45f43e09a88b979b29e0f23 2024-12-08T00:16:00,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/f13e0f73a45f43e09a88b979b29e0f23, entries=150, sequenceid=739, filesize=12.0 K 2024-12-08T00:16:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/b73662665c864dc58bc17f27aac282ca as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b73662665c864dc58bc17f27aac282ca 2024-12-08T00:16:00,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b73662665c864dc58bc17f27aac282ca, entries=150, sequenceid=739, filesize=12.0 K 2024-12-08T00:16:00,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~279.98 KB/286704, currentSize=87.22 KB/89310 for 7404e7668f405a8889810821ba8518dd in 872ms, sequenceid=739, compaction requested=true 2024-12-08T00:16:00,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:16:00,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:00,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:00,239 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:00,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:00,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:00,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:00,239 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:00,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:00,239 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111976 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:00,239 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:00,239 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/B is initiating minor compaction (all files) 2024-12-08T00:16:00,239 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/A is initiating minor compaction (all files) 2024-12-08T00:16:00,239 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/B in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:00,239 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/A in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:00,240 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fd2668d514ec4c2bbf591dbcd55fc25d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/463ab095e1dd4cc09578fa21ab54ed7b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/09e79c45fa024767a31f92e2cd0e4f16] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=109.4 K 2024-12-08T00:16:00,240 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/75e046df12704dc19d8017bc28320001, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/dbb32111b28c4487bec3051cabd04d5f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/f13e0f73a45f43e09a88b979b29e0f23] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=36.7 K 2024-12-08T00:16:00,240 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:00,240 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fd2668d514ec4c2bbf591dbcd55fc25d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/463ab095e1dd4cc09578fa21ab54ed7b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/09e79c45fa024767a31f92e2cd0e4f16] 2024-12-08T00:16:00,240 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e046df12704dc19d8017bc28320001, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616954841 2024-12-08T00:16:00,240 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd2668d514ec4c2bbf591dbcd55fc25d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616954841 2024-12-08T00:16:00,240 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting dbb32111b28c4487bec3051cabd04d5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733616957272 2024-12-08T00:16:00,240 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 463ab095e1dd4cc09578fa21ab54ed7b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733616957272 2024-12-08T00:16:00,240 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e79c45fa024767a31f92e2cd0e4f16, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=739, earliestPutTs=1733616959339 2024-12-08T00:16:00,240 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting f13e0f73a45f43e09a88b979b29e0f23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=739, earliestPutTs=1733616959360 2024-12-08T00:16:00,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T00:16:00,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:00,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T00:16:00,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:16:00,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:00,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:16:00,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:00,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:16:00,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:00,248 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:00,248 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/73c7186534e2405388aef0a2ef83c2b0 is 50, key is test_row_0/B:col10/1733616959365/Put/seqid=0 2024-12-08T00:16:00,257 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087a7fe9e363774c29af5aa56817bc2677_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616959395/Put/seqid=0 2024-12-08T00:16:00,259 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412085c1c7cc14df540b48794c0b2cb8ad482_7404e7668f405a8889810821ba8518dd store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:00,262 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412085c1c7cc14df540b48794c0b2cb8ad482_7404e7668f405a8889810821ba8518dd, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:00,263 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085c1c7cc14df540b48794c0b2cb8ad482_7404e7668f405a8889810821ba8518dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:00,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742211_1387 (size=13119) 2024-12-08T00:16:00,270 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/73c7186534e2405388aef0a2ef83c2b0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/73c7186534e2405388aef0a2ef83c2b0 2024-12-08T00:16:00,283 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/B of 7404e7668f405a8889810821ba8518dd into 73c7186534e2405388aef0a2ef83c2b0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:00,283 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:16:00,283 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/B, priority=13, startTime=1733616960239; duration=0sec 2024-12-08T00:16:00,283 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:00,283 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:B 2024-12-08T00:16:00,283 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:00,283 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:00,284 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/C is initiating minor compaction (all files) 2024-12-08T00:16:00,284 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/C in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:00,284 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/19c62ad56d334cfb9a173bb4f176aecb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/143357098f144e7ea7933fda10add313, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b73662665c864dc58bc17f27aac282ca] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=36.7 K 2024-12-08T00:16:00,284 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c62ad56d334cfb9a173bb4f176aecb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733616954841 2024-12-08T00:16:00,284 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 143357098f144e7ea7933fda10add313, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733616957272 2024-12-08T00:16:00,284 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting b73662665c864dc58bc17f27aac282ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=739, earliestPutTs=1733616959360 2024-12-08T00:16:00,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742212_1388 (size=4469) 2024-12-08T00:16:00,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742213_1389 (size=12454) 2024-12-08T00:16:00,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:00,293 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:00,294 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/0c0af177a16e4bb7840f74bdb192e983 is 50, key is test_row_0/C:col10/1733616959365/Put/seqid=0 2024-12-08T00:16:00,296 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087a7fe9e363774c29af5aa56817bc2677_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087a7fe9e363774c29af5aa56817bc2677_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:00,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742214_1390 (size=13119) 2024-12-08T00:16:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6c84d3e9c90248f8b815a84f4793357f, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6c84d3e9c90248f8b815a84f4793357f is 175, key is test_row_0/A:col10/1733616959395/Put/seqid=0 2024-12-08T00:16:00,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742215_1391 (size=31255) 2024-12-08T00:16:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:00,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. as already flushing 2024-12-08T00:16:00,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617020582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 360 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617020592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617020592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 386 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617020593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,695 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#A#compaction#319 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:00,695 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/de87d366962346518e3864b009929965 is 175, key is test_row_0/A:col10/1733616959365/Put/seqid=0 2024-12-08T00:16:00,702 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=755, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6c84d3e9c90248f8b815a84f4793357f 2024-12-08T00:16:00,703 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/0c0af177a16e4bb7840f74bdb192e983 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/0c0af177a16e4bb7840f74bdb192e983 2024-12-08T00:16:00,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617020701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,715 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/C of 7404e7668f405a8889810821ba8518dd into 0c0af177a16e4bb7840f74bdb192e983(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:00,715 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:16:00,715 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/C, priority=13, startTime=1733616960239; duration=0sec 2024-12-08T00:16:00,715 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:00,715 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:C 2024-12-08T00:16:00,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742216_1392 (size=32073) 2024-12-08T00:16:00,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/3e46b95a02524cac8a401416702851d9 is 50, key is test_row_0/B:col10/1733616959395/Put/seqid=0 2024-12-08T00:16:00,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 361 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617020711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617020711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 387 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617020712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,725 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/de87d366962346518e3864b009929965 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/de87d366962346518e3864b009929965 2024-12-08T00:16:00,731 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/A of 7404e7668f405a8889810821ba8518dd into de87d366962346518e3864b009929965(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:00,731 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:16:00,731 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/A, priority=13, startTime=1733616960238; duration=0sec 2024-12-08T00:16:00,731 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:00,731 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:A 2024-12-08T00:16:00,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742217_1393 (size=12301) 2024-12-08T00:16:00,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617020921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 362 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617020930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617020931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 388 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617020931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:00,964 INFO [Thread-1356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1356. 2024-12-08T00:16:00,964 DEBUG [Thread-1356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:00,964 DEBUG [Thread-1356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,964 DEBUG [Thread-1356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:00,967 INFO [Thread-1364 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1364. 2024-12-08T00:16:00,967 DEBUG [Thread-1364 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:00,967 DEBUG [Thread-1364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,967 DEBUG [Thread-1364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:00,969 INFO [Thread-1358 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1358. 2024-12-08T00:16:00,969 DEBUG [Thread-1358 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:00,969 DEBUG [Thread-1358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,969 DEBUG [Thread-1358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:00,973 INFO [Thread-1360 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1360. 2024-12-08T00:16:00,973 DEBUG [Thread-1360 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:00,973 DEBUG [Thread-1360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,973 DEBUG [Thread-1360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:00,973 INFO [Thread-1362 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1362. 2024-12-08T00:16:00,973 DEBUG [Thread-1362 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:00,973 DEBUG [Thread-1362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,973 DEBUG [Thread-1362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:00,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:01,146 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/3e46b95a02524cac8a401416702851d9 2024-12-08T00:16:01,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/da0c689add8040b4b9ac7a570a7d42a4 is 50, key is test_row_0/C:col10/1733616959395/Put/seqid=0 2024-12-08T00:16:01,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742218_1394 (size=12301) 2024-12-08T00:16:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T00:16:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1733617021241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 363 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1733617021241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32830 deadline: 1733617021241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:01,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 389 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32828 deadline: 1733617021242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:01,563 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/da0c689add8040b4b9ac7a570a7d42a4 2024-12-08T00:16:01,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/6c84d3e9c90248f8b815a84f4793357f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6c84d3e9c90248f8b815a84f4793357f 2024-12-08T00:16:01,573 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6c84d3e9c90248f8b815a84f4793357f, entries=150, sequenceid=755, filesize=30.5 K 2024-12-08T00:16:01,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/3e46b95a02524cac8a401416702851d9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3e46b95a02524cac8a401416702851d9 2024-12-08T00:16:01,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3e46b95a02524cac8a401416702851d9, entries=150, sequenceid=755, filesize=12.0 K 2024-12-08T00:16:01,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/da0c689add8040b4b9ac7a570a7d42a4 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/da0c689add8040b4b9ac7a570a7d42a4 2024-12-08T00:16:01,583 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/da0c689add8040b4b9ac7a570a7d42a4, entries=150, sequenceid=755, filesize=12.0 K 2024-12-08T00:16:01,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 7404e7668f405a8889810821ba8518dd in 1340ms, sequenceid=755, compaction requested=false 2024-12-08T00:16:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:16:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-08T00:16:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-08T00:16:01,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-08T00:16:01,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.5520 sec 2024-12-08T00:16:01,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 4.5570 sec 2024-12-08T00:16:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:01,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:16:01,753 INFO [Thread-1345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1345. 2024-12-08T00:16:01,753 INFO [Thread-1353 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1353. 2024-12-08T00:16:01,753 DEBUG [Thread-1345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:01,753 INFO [Thread-1351 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1351. 2024-12-08T00:16:01,753 DEBUG [Thread-1345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:01,753 DEBUG [Thread-1353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:01,753 DEBUG [Thread-1351 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:01,753 DEBUG [Thread-1353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:01,753 DEBUG [Thread-1351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:01,753 DEBUG [Thread-1351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:01,753 DEBUG [Thread-1345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:01,753 DEBUG [Thread-1353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:01,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:01,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:01,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:01,754 INFO [Thread-1349 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1349. 2024-12-08T00:16:01,755 DEBUG [Thread-1349 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:01,755 DEBUG [Thread-1349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:01,755 DEBUG [Thread-1349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:16:01,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:01,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616961751, max timestamp=1733616961753Num uniques 859; 2024-12-08T00:16:01,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616961751, max timestamp=1733616961753Num uniques 859; 2024-12-08T00:16:01,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:01,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:01,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616961751, max timestamp=1733616961753Num uniques 859; 2024-12-08T00:16:01,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:16:01,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:01,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:16:01,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:01,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081dd5eff3ea9440ac87fa44bc0b719ba8_7404e7668f405a8889810821ba8518dd is 50, key is test_row_0/A:col10/1733616961751/Put/seqid=0 2024-12-08T00:16:01,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742219_1395 (size=12454) 2024-12-08T00:16:02,166 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:02,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081dd5eff3ea9440ac87fa44bc0b719ba8_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081dd5eff3ea9440ac87fa44bc0b719ba8_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:02,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fe2d9111fe294cfcb9c51fd1ac2526d1, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:02,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fe2d9111fe294cfcb9c51fd1ac2526d1 is 175, key is test_row_0/A:col10/1733616961751/Put/seqid=0 2024-12-08T00:16:02,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742220_1396 (size=31255) 2024-12-08T00:16:02,576 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=779, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fe2d9111fe294cfcb9c51fd1ac2526d1 2024-12-08T00:16:02,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/6f3de0784bf343cdbb9cd124c7345fb8 is 50, key is test_row_0/B:col10/1733616961751/Put/seqid=0 2024-12-08T00:16:02,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742221_1397 (size=12301) 2024-12-08T00:16:02,684 INFO [Thread-1347 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1347. 2024-12-08T00:16:02,684 DEBUG [Thread-1347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:02,684 DEBUG [Thread-1347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:02,684 DEBUG [Thread-1347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:02,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:02,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=779 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/6f3de0784bf343cdbb9cd124c7345fb8 2024-12-08T00:16:02,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/54efbcb1bd2f4f6db7b9b685bbcf0b0e is 50, key is test_row_0/C:col10/1733616961751/Put/seqid=0 2024-12-08T00:16:02,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742222_1398 (size=12301) 2024-12-08T00:16:03,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=779 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/54efbcb1bd2f4f6db7b9b685bbcf0b0e 2024-12-08T00:16:03,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/fe2d9111fe294cfcb9c51fd1ac2526d1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fe2d9111fe294cfcb9c51fd1ac2526d1 2024-12-08T00:16:03,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fe2d9111fe294cfcb9c51fd1ac2526d1, entries=150, sequenceid=779, filesize=30.5 K 2024-12-08T00:16:03,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/6f3de0784bf343cdbb9cd124c7345fb8 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6f3de0784bf343cdbb9cd124c7345fb8 2024-12-08T00:16:03,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6f3de0784bf343cdbb9cd124c7345fb8, entries=150, sequenceid=779, filesize=12.0 K 2024-12-08T00:16:03,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/54efbcb1bd2f4f6db7b9b685bbcf0b0e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/54efbcb1bd2f4f6db7b9b685bbcf0b0e 2024-12-08T00:16:03,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/54efbcb1bd2f4f6db7b9b685bbcf0b0e, entries=150, sequenceid=779, filesize=12.0 K 2024-12-08T00:16:03,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=6.71 KB/6870 for 7404e7668f405a8889810821ba8518dd in 1661ms, sequenceid=779, compaction requested=true 2024-12-08T00:16:03,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:16:03,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:03,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:03,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:03,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:03,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7404e7668f405a8889810821ba8518dd:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:03,413 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:03,413 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:03,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:03,414 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:03,414 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:03,414 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/B is initiating minor compaction (all files) 2024-12-08T00:16:03,414 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/A is initiating minor compaction (all files) 2024-12-08T00:16:03,414 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/B in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:03,414 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/A in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:03,414 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/73c7186534e2405388aef0a2ef83c2b0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3e46b95a02524cac8a401416702851d9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6f3de0784bf343cdbb9cd124c7345fb8] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=36.8 K 2024-12-08T00:16:03,414 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/de87d366962346518e3864b009929965, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6c84d3e9c90248f8b815a84f4793357f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fe2d9111fe294cfcb9c51fd1ac2526d1] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=92.4 K 2024-12-08T00:16:03,414 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:03,414 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/de87d366962346518e3864b009929965, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6c84d3e9c90248f8b815a84f4793357f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fe2d9111fe294cfcb9c51fd1ac2526d1] 2024-12-08T00:16:03,415 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 73c7186534e2405388aef0a2ef83c2b0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=739, earliestPutTs=1733616959360 2024-12-08T00:16:03,415 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting de87d366962346518e3864b009929965, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=739, earliestPutTs=1733616959360 2024-12-08T00:16:03,415 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e46b95a02524cac8a401416702851d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=755, earliestPutTs=1733616959393 2024-12-08T00:16:03,415 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c84d3e9c90248f8b815a84f4793357f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=755, earliestPutTs=1733616959393 2024-12-08T00:16:03,415 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f3de0784bf343cdbb9cd124c7345fb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=779, earliestPutTs=1733616961751 2024-12-08T00:16:03,415 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe2d9111fe294cfcb9c51fd1ac2526d1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=779, earliestPutTs=1733616961751 2024-12-08T00:16:03,422 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:03,422 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:03,422 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/a6f9876aeddf4b118736ea490dc8f7d0 is 50, key is test_row_0/B:col10/1733616961751/Put/seqid=0 2024-12-08T00:16:03,424 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208a8d83452c29149be92e6032f1f37c513_7404e7668f405a8889810821ba8518dd store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:03,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742223_1399 (size=13221) 2024-12-08T00:16:03,427 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208a8d83452c29149be92e6032f1f37c513_7404e7668f405a8889810821ba8518dd, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:03,427 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a8d83452c29149be92e6032f1f37c513_7404e7668f405a8889810821ba8518dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:03,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742224_1400 (size=4469) 2024-12-08T00:16:03,830 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/a6f9876aeddf4b118736ea490dc8f7d0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/a6f9876aeddf4b118736ea490dc8f7d0 2024-12-08T00:16:03,831 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:03,831 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/0b390d4e7dd045c49da9723428184292 is 175, key is test_row_0/A:col10/1733616961751/Put/seqid=0 2024-12-08T00:16:03,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742225_1401 (size=32175) 2024-12-08T00:16:03,834 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/B of 7404e7668f405a8889810821ba8518dd into a6f9876aeddf4b118736ea490dc8f7d0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:03,834 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:16:03,834 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/B, priority=13, startTime=1733616963413; duration=0sec 2024-12-08T00:16:03,835 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:03,835 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:B 2024-12-08T00:16:03,835 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:03,835 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:03,835 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 7404e7668f405a8889810821ba8518dd/C is initiating minor compaction (all files) 2024-12-08T00:16:03,836 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7404e7668f405a8889810821ba8518dd/C in TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:03,836 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/0c0af177a16e4bb7840f74bdb192e983, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/da0c689add8040b4b9ac7a570a7d42a4, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/54efbcb1bd2f4f6db7b9b685bbcf0b0e] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp, totalSize=36.8 K 2024-12-08T00:16:03,836 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c0af177a16e4bb7840f74bdb192e983, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=739, earliestPutTs=1733616959360 2024-12-08T00:16:03,836 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting da0c689add8040b4b9ac7a570a7d42a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=755, earliestPutTs=1733616959393 2024-12-08T00:16:03,836 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 54efbcb1bd2f4f6db7b9b685bbcf0b0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=779, earliestPutTs=1733616961751 2024-12-08T00:16:03,844 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7404e7668f405a8889810821ba8518dd#C#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:03,844 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/8e3191bc2be5488d89bc203e710351eb is 50, key is test_row_0/C:col10/1733616961751/Put/seqid=0 2024-12-08T00:16:03,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742226_1402 (size=13221) 2024-12-08T00:16:04,239 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/0b390d4e7dd045c49da9723428184292 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0b390d4e7dd045c49da9723428184292 2024-12-08T00:16:04,243 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/A of 7404e7668f405a8889810821ba8518dd into 0b390d4e7dd045c49da9723428184292(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:04,243 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:16:04,243 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/A, priority=13, startTime=1733616963413; duration=0sec 2024-12-08T00:16:04,243 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:04,243 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:A 2024-12-08T00:16:04,251 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/8e3191bc2be5488d89bc203e710351eb as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/8e3191bc2be5488d89bc203e710351eb 2024-12-08T00:16:04,256 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7404e7668f405a8889810821ba8518dd/C of 7404e7668f405a8889810821ba8518dd into 8e3191bc2be5488d89bc203e710351eb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:04,256 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7404e7668f405a8889810821ba8518dd: 2024-12-08T00:16:04,256 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., storeName=7404e7668f405a8889810821ba8518dd/C, priority=13, startTime=1733616963413; duration=0sec 2024-12-08T00:16:04,256 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:04,256 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7404e7668f405a8889810821ba8518dd:C 2024-12-08T00:16:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T00:16:05,232 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 304 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 331 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1480 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4440 rows 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1484 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4445 rows 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1477 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4430 rows 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1482 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4446 rows 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1486 2024-12-08T00:16:05,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4457 rows 2024-12-08T00:16:05,232 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T00:16:05,233 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T00:16:05,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:05,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:05,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:05,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T00:16:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T00:16:05,239 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616965239"}]},"ts":"1733616965239"} 2024-12-08T00:16:05,241 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T00:16:05,241 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T00:16:05,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T00:16:05,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, UNASSIGN}] 2024-12-08T00:16:05,245 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, UNASSIGN 2024-12-08T00:16:05,246 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=7404e7668f405a8889810821ba8518dd, regionState=CLOSING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:05,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, UNASSIGN because future has completed 2024-12-08T00:16:05,247 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T00:16:05,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:16:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T00:16:05,400 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:05,401 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T00:16:05,401 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 7404e7668f405a8889810821ba8518dd, disabling compactions & flushes 2024-12-08T00:16:05,401 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:05,401 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:05,401 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. after waiting 0 ms 2024-12-08T00:16:05,401 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:05,401 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 7404e7668f405a8889810821ba8518dd 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-08T00:16:05,402 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=A 2024-12-08T00:16:05,402 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:05,402 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=B 2024-12-08T00:16:05,402 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:05,402 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7404e7668f405a8889810821ba8518dd, store=C 2024-12-08T00:16:05,402 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:05,410 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087e16f16754904a5297b54fef428247f0_7404e7668f405a8889810821ba8518dd is 50, key is test_row_2/A:col10/1733616962682/Put/seqid=0 2024-12-08T00:16:05,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742227_1403 (size=7374) 2024-12-08T00:16:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T00:16:05,816 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:05,819 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087e16f16754904a5297b54fef428247f0_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087e16f16754904a5297b54fef428247f0_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:05,820 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/46ef837aa09846d0ac35f1d6b492450b, store: [table=TestAcidGuarantees family=A region=7404e7668f405a8889810821ba8518dd] 2024-12-08T00:16:05,820 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/46ef837aa09846d0ac35f1d6b492450b is 175, key is test_row_2/A:col10/1733616962682/Put/seqid=0 2024-12-08T00:16:05,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742228_1404 (size=13865) 2024-12-08T00:16:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T00:16:06,224 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=786, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/46ef837aa09846d0ac35f1d6b492450b 2024-12-08T00:16:06,232 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/acae3bb9798444e3999efd252d3c79b3 is 50, key is test_row_2/B:col10/1733616962682/Put/seqid=0 2024-12-08T00:16:06,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742229_1405 (size=7415) 2024-12-08T00:16:06,243 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T00:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T00:16:06,636 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/acae3bb9798444e3999efd252d3c79b3 2024-12-08T00:16:06,642 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/376ccd57030b4e9bba0b56c4e4ec4312 is 50, key is test_row_2/C:col10/1733616962682/Put/seqid=0 2024-12-08T00:16:06,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742230_1406 (size=7415) 2024-12-08T00:16:07,047 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/376ccd57030b4e9bba0b56c4e4ec4312 2024-12-08T00:16:07,058 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/A/46ef837aa09846d0ac35f1d6b492450b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/46ef837aa09846d0ac35f1d6b492450b 2024-12-08T00:16:07,061 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/46ef837aa09846d0ac35f1d6b492450b, entries=50, sequenceid=786, filesize=13.5 K 2024-12-08T00:16:07,062 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/B/acae3bb9798444e3999efd252d3c79b3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/acae3bb9798444e3999efd252d3c79b3 2024-12-08T00:16:07,066 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/acae3bb9798444e3999efd252d3c79b3, entries=50, sequenceid=786, filesize=7.2 K 2024-12-08T00:16:07,066 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/.tmp/C/376ccd57030b4e9bba0b56c4e4ec4312 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/376ccd57030b4e9bba0b56c4e4ec4312 2024-12-08T00:16:07,070 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/376ccd57030b4e9bba0b56c4e4ec4312, entries=50, sequenceid=786, filesize=7.2 K 2024-12-08T00:16:07,071 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 7404e7668f405a8889810821ba8518dd in 1670ms, sequenceid=786, compaction requested=false 2024-12-08T00:16:07,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fb6cf85436644c47a743bb5331497dba, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/76925bbaa8764652abc3531fbb4020d8, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/b8ea19b694684bfa80fe32e39f3a0013, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6efaa04cf75b4800a7dcfdc9af6a38b7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/53e8d907f393467eb9c2fa916c0b2727, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f76997f83fbe4e788be77926fab0de6f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/10812f38e9234a45a73f8d41c4a4f4cd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/4fb3e4af23e34dc7a474aea7d49d71c7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/aefba380d4634aefaa3d5f562c0404ea, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/2bbdb3bccfbc4422a4835fdbcec825cc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0bc715b24d2d4f85b21e360d7b1c2df5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d7364370195c48f6a822b80648286305, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/1ce05430ffae4839b715cc31a690ab9a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f0f8ed576f6e41daa797001dfefecb73, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/13df0183b38145879c92a34f5223796d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/708050dca22d49f4834936a9fd1e5010, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fec90fc54d9442809a2adcf7e555a25f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6111f5c366ef4afcbaeb7b5497ab534c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/bb3dc368c3c641d08073a448b7075f59, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d76db06456d64180a17edc6e463d7c89, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fd2668d514ec4c2bbf591dbcd55fc25d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6de7b917c69b4d81a3e74dde0b800d34, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/463ab095e1dd4cc09578fa21ab54ed7b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/09e79c45fa024767a31f92e2cd0e4f16, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/de87d366962346518e3864b009929965, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6c84d3e9c90248f8b815a84f4793357f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fe2d9111fe294cfcb9c51fd1ac2526d1] to archive 2024-12-08T00:16:07,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:16:07,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fb6cf85436644c47a743bb5331497dba to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fb6cf85436644c47a743bb5331497dba 2024-12-08T00:16:07,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/76925bbaa8764652abc3531fbb4020d8 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/76925bbaa8764652abc3531fbb4020d8 2024-12-08T00:16:07,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/b8ea19b694684bfa80fe32e39f3a0013 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/b8ea19b694684bfa80fe32e39f3a0013 2024-12-08T00:16:07,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6efaa04cf75b4800a7dcfdc9af6a38b7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6efaa04cf75b4800a7dcfdc9af6a38b7 2024-12-08T00:16:07,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/53e8d907f393467eb9c2fa916c0b2727 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/53e8d907f393467eb9c2fa916c0b2727 2024-12-08T00:16:07,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f76997f83fbe4e788be77926fab0de6f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f76997f83fbe4e788be77926fab0de6f 2024-12-08T00:16:07,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/10812f38e9234a45a73f8d41c4a4f4cd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/10812f38e9234a45a73f8d41c4a4f4cd 2024-12-08T00:16:07,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/4fb3e4af23e34dc7a474aea7d49d71c7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/4fb3e4af23e34dc7a474aea7d49d71c7 2024-12-08T00:16:07,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/aefba380d4634aefaa3d5f562c0404ea to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/aefba380d4634aefaa3d5f562c0404ea 2024-12-08T00:16:07,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/2bbdb3bccfbc4422a4835fdbcec825cc to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/2bbdb3bccfbc4422a4835fdbcec825cc 2024-12-08T00:16:07,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0bc715b24d2d4f85b21e360d7b1c2df5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0bc715b24d2d4f85b21e360d7b1c2df5 2024-12-08T00:16:07,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d7364370195c48f6a822b80648286305 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d7364370195c48f6a822b80648286305 2024-12-08T00:16:07,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/1ce05430ffae4839b715cc31a690ab9a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/1ce05430ffae4839b715cc31a690ab9a 2024-12-08T00:16:07,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f0f8ed576f6e41daa797001dfefecb73 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/f0f8ed576f6e41daa797001dfefecb73 2024-12-08T00:16:07,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/13df0183b38145879c92a34f5223796d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/13df0183b38145879c92a34f5223796d 2024-12-08T00:16:07,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/708050dca22d49f4834936a9fd1e5010 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/708050dca22d49f4834936a9fd1e5010 2024-12-08T00:16:07,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fec90fc54d9442809a2adcf7e555a25f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fec90fc54d9442809a2adcf7e555a25f 2024-12-08T00:16:07,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6111f5c366ef4afcbaeb7b5497ab534c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6111f5c366ef4afcbaeb7b5497ab534c 2024-12-08T00:16:07,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/bb3dc368c3c641d08073a448b7075f59 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/bb3dc368c3c641d08073a448b7075f59 2024-12-08T00:16:07,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d76db06456d64180a17edc6e463d7c89 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/d76db06456d64180a17edc6e463d7c89 2024-12-08T00:16:07,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fd2668d514ec4c2bbf591dbcd55fc25d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fd2668d514ec4c2bbf591dbcd55fc25d 2024-12-08T00:16:07,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6de7b917c69b4d81a3e74dde0b800d34 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6de7b917c69b4d81a3e74dde0b800d34 2024-12-08T00:16:07,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/463ab095e1dd4cc09578fa21ab54ed7b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/463ab095e1dd4cc09578fa21ab54ed7b 2024-12-08T00:16:07,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/09e79c45fa024767a31f92e2cd0e4f16 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/09e79c45fa024767a31f92e2cd0e4f16 2024-12-08T00:16:07,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/de87d366962346518e3864b009929965 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/de87d366962346518e3864b009929965 2024-12-08T00:16:07,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6c84d3e9c90248f8b815a84f4793357f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/6c84d3e9c90248f8b815a84f4793357f 2024-12-08T00:16:07,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fe2d9111fe294cfcb9c51fd1ac2526d1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/fe2d9111fe294cfcb9c51fd1ac2526d1 2024-12-08T00:16:07,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/e93d56eccec34660b4e84dd94f32b55d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/fccdb3ffd501482abe3974cd41c08b6e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6e0c6c534b1947e39a83260f5fafeba5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6338b3d4733a4a86bbe2dac705468213, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/4cf2031bf73945b58aabf9ade83abf3c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/7e219ba0c9184004b8f3ea72e3c389f6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/49ff60bd608449c2a0480fd76e134781, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ffad9704288d4bb5bfabba1c01d71c3c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/bf39b5a4125346449e4278f45d22f257, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/549a3dc7fa4b4be7a9654b709cc1eb84, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/a867a8cda14042d7a91d0fe042181cb5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/23719d7d142447ceba090650cfe6db67, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/eee6f5dcd7024f68ae968196467cdb37, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/8d9038f5767f451fab8abffd77420cbe, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3210307c7dcc4efb942fce65dbedc811, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/c21206e878f64937aab1680587f00ece, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/415d9ee246ee45e08b6a6e9ce7b19483, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/2952c8ef7ace4de7929f90a2d42c4f0a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/1c9a12f04ab9450d968b3c8eee7203a4, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/d1787433fa14476388b4e055d3412953, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/75e046df12704dc19d8017bc28320001, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ed56b56c562542709c1b70ab389e6b09, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/dbb32111b28c4487bec3051cabd04d5f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/73c7186534e2405388aef0a2ef83c2b0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/f13e0f73a45f43e09a88b979b29e0f23, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3e46b95a02524cac8a401416702851d9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6f3de0784bf343cdbb9cd124c7345fb8] to archive 2024-12-08T00:16:07,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:16:07,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/e93d56eccec34660b4e84dd94f32b55d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/e93d56eccec34660b4e84dd94f32b55d 2024-12-08T00:16:07,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/fccdb3ffd501482abe3974cd41c08b6e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/fccdb3ffd501482abe3974cd41c08b6e 2024-12-08T00:16:07,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6e0c6c534b1947e39a83260f5fafeba5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6e0c6c534b1947e39a83260f5fafeba5 2024-12-08T00:16:07,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6338b3d4733a4a86bbe2dac705468213 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6338b3d4733a4a86bbe2dac705468213 2024-12-08T00:16:07,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/4cf2031bf73945b58aabf9ade83abf3c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/4cf2031bf73945b58aabf9ade83abf3c 2024-12-08T00:16:07,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/7e219ba0c9184004b8f3ea72e3c389f6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/7e219ba0c9184004b8f3ea72e3c389f6 2024-12-08T00:16:07,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/49ff60bd608449c2a0480fd76e134781 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/49ff60bd608449c2a0480fd76e134781 2024-12-08T00:16:07,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ffad9704288d4bb5bfabba1c01d71c3c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ffad9704288d4bb5bfabba1c01d71c3c 2024-12-08T00:16:07,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/bf39b5a4125346449e4278f45d22f257 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/bf39b5a4125346449e4278f45d22f257 2024-12-08T00:16:07,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/549a3dc7fa4b4be7a9654b709cc1eb84 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/549a3dc7fa4b4be7a9654b709cc1eb84 2024-12-08T00:16:07,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/a867a8cda14042d7a91d0fe042181cb5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/a867a8cda14042d7a91d0fe042181cb5 2024-12-08T00:16:07,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/23719d7d142447ceba090650cfe6db67 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/23719d7d142447ceba090650cfe6db67 2024-12-08T00:16:07,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/eee6f5dcd7024f68ae968196467cdb37 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/eee6f5dcd7024f68ae968196467cdb37 2024-12-08T00:16:07,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/8d9038f5767f451fab8abffd77420cbe to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/8d9038f5767f451fab8abffd77420cbe 2024-12-08T00:16:07,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3210307c7dcc4efb942fce65dbedc811 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3210307c7dcc4efb942fce65dbedc811 2024-12-08T00:16:07,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/c21206e878f64937aab1680587f00ece to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/c21206e878f64937aab1680587f00ece 2024-12-08T00:16:07,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/415d9ee246ee45e08b6a6e9ce7b19483 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/415d9ee246ee45e08b6a6e9ce7b19483 2024-12-08T00:16:07,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/2952c8ef7ace4de7929f90a2d42c4f0a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/2952c8ef7ace4de7929f90a2d42c4f0a 2024-12-08T00:16:07,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/1c9a12f04ab9450d968b3c8eee7203a4 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/1c9a12f04ab9450d968b3c8eee7203a4 2024-12-08T00:16:07,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/d1787433fa14476388b4e055d3412953 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/d1787433fa14476388b4e055d3412953 2024-12-08T00:16:07,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/75e046df12704dc19d8017bc28320001 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/75e046df12704dc19d8017bc28320001 2024-12-08T00:16:07,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ed56b56c562542709c1b70ab389e6b09 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/ed56b56c562542709c1b70ab389e6b09 2024-12-08T00:16:07,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/dbb32111b28c4487bec3051cabd04d5f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/dbb32111b28c4487bec3051cabd04d5f 2024-12-08T00:16:07,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/73c7186534e2405388aef0a2ef83c2b0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/73c7186534e2405388aef0a2ef83c2b0 2024-12-08T00:16:07,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/f13e0f73a45f43e09a88b979b29e0f23 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/f13e0f73a45f43e09a88b979b29e0f23 2024-12-08T00:16:07,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3e46b95a02524cac8a401416702851d9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/3e46b95a02524cac8a401416702851d9 2024-12-08T00:16:07,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6f3de0784bf343cdbb9cd124c7345fb8 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/6f3de0784bf343cdbb9cd124c7345fb8 2024-12-08T00:16:07,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/be028f2857104140b3c478df97557a14, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e2c117f84cab438cae9aa29795215b2d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3d01a0e0403549a5aa8cfcc565302571, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6b938e68f1b34da794d43f6779bbd270, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/268a7dd605044630945dd9bf85f9b707, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6bd97964636546c3abea1e59d320b734, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e48408cbc3234d979ec6781aabaae06d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b1c6d95c96304b77b8c881cf451d7acc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/826c0365f9a34a519c83e04057753b88, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9dac6461cbb14f84a997a0270fae3e25, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/1fe067f1abf2463d88a169a06b19385a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/fd99674c2b5b42cd88a2d4ea57f4bde7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/708f6203ecd04c9bbeda87c1ef348813, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e13b8fdacc224d21a348ab7e43f5b6b3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/83025f3585654cde88b1df702edbdadc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3b698d86a71046fe84bd969253271289, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/bccbeb3f5c5946f786a5c002c7571b40, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9bced484a44d465d880ccfc21a2b1515, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/ee614ee6b00d461c991eebb2dca25afb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3124b4b595894bd19078d18dd2d1887b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/19c62ad56d334cfb9a173bb4f176aecb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9f8cc70efac24d128faf1e15455d315b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/143357098f144e7ea7933fda10add313, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/0c0af177a16e4bb7840f74bdb192e983, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b73662665c864dc58bc17f27aac282ca, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/da0c689add8040b4b9ac7a570a7d42a4, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/54efbcb1bd2f4f6db7b9b685bbcf0b0e] to archive 2024-12-08T00:16:07,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:16:07,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/be028f2857104140b3c478df97557a14 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/be028f2857104140b3c478df97557a14 2024-12-08T00:16:07,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e2c117f84cab438cae9aa29795215b2d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e2c117f84cab438cae9aa29795215b2d 2024-12-08T00:16:07,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3d01a0e0403549a5aa8cfcc565302571 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3d01a0e0403549a5aa8cfcc565302571 2024-12-08T00:16:07,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6b938e68f1b34da794d43f6779bbd270 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6b938e68f1b34da794d43f6779bbd270 2024-12-08T00:16:07,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/268a7dd605044630945dd9bf85f9b707 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/268a7dd605044630945dd9bf85f9b707 2024-12-08T00:16:07,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6bd97964636546c3abea1e59d320b734 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/6bd97964636546c3abea1e59d320b734 2024-12-08T00:16:07,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e48408cbc3234d979ec6781aabaae06d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e48408cbc3234d979ec6781aabaae06d 2024-12-08T00:16:07,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b1c6d95c96304b77b8c881cf451d7acc to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b1c6d95c96304b77b8c881cf451d7acc 2024-12-08T00:16:07,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/826c0365f9a34a519c83e04057753b88 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/826c0365f9a34a519c83e04057753b88 2024-12-08T00:16:07,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9dac6461cbb14f84a997a0270fae3e25 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9dac6461cbb14f84a997a0270fae3e25 2024-12-08T00:16:07,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/1fe067f1abf2463d88a169a06b19385a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/1fe067f1abf2463d88a169a06b19385a 2024-12-08T00:16:07,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/fd99674c2b5b42cd88a2d4ea57f4bde7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/fd99674c2b5b42cd88a2d4ea57f4bde7 2024-12-08T00:16:07,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/708f6203ecd04c9bbeda87c1ef348813 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/708f6203ecd04c9bbeda87c1ef348813 2024-12-08T00:16:07,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e13b8fdacc224d21a348ab7e43f5b6b3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/e13b8fdacc224d21a348ab7e43f5b6b3 2024-12-08T00:16:07,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/83025f3585654cde88b1df702edbdadc to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/83025f3585654cde88b1df702edbdadc 2024-12-08T00:16:07,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3b698d86a71046fe84bd969253271289 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3b698d86a71046fe84bd969253271289 2024-12-08T00:16:07,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/bccbeb3f5c5946f786a5c002c7571b40 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/bccbeb3f5c5946f786a5c002c7571b40 2024-12-08T00:16:07,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9bced484a44d465d880ccfc21a2b1515 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9bced484a44d465d880ccfc21a2b1515 2024-12-08T00:16:07,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/ee614ee6b00d461c991eebb2dca25afb to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/ee614ee6b00d461c991eebb2dca25afb 2024-12-08T00:16:07,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3124b4b595894bd19078d18dd2d1887b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/3124b4b595894bd19078d18dd2d1887b 2024-12-08T00:16:07,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/19c62ad56d334cfb9a173bb4f176aecb to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/19c62ad56d334cfb9a173bb4f176aecb 2024-12-08T00:16:07,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9f8cc70efac24d128faf1e15455d315b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/9f8cc70efac24d128faf1e15455d315b 2024-12-08T00:16:07,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/143357098f144e7ea7933fda10add313 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/143357098f144e7ea7933fda10add313 2024-12-08T00:16:07,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/0c0af177a16e4bb7840f74bdb192e983 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/0c0af177a16e4bb7840f74bdb192e983 2024-12-08T00:16:07,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b73662665c864dc58bc17f27aac282ca to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/b73662665c864dc58bc17f27aac282ca 2024-12-08T00:16:07,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/da0c689add8040b4b9ac7a570a7d42a4 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/da0c689add8040b4b9ac7a570a7d42a4 2024-12-08T00:16:07,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/54efbcb1bd2f4f6db7b9b685bbcf0b0e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/54efbcb1bd2f4f6db7b9b685bbcf0b0e 2024-12-08T00:16:07,150 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/recovered.edits/789.seqid, newMaxSeqId=789, maxSeqId=4 2024-12-08T00:16:07,150 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. 2024-12-08T00:16:07,150 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 7404e7668f405a8889810821ba8518dd: Waiting for close lock at 1733616965401Running coprocessor pre-close hooks at 1733616965401Disabling compacts and flushes for region at 1733616965401Disabling writes for close at 1733616965401Obtaining lock to block concurrent updates at 1733616965401Preparing flush snapshotting stores in 7404e7668f405a8889810821ba8518dd at 1733616965401Finished memstore snapshotting TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1733616965403 (+2 ms)Flushing stores of TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd. at 1733616965404 (+1 ms)Flushing 7404e7668f405a8889810821ba8518dd/A: creating writer at 1733616965404Flushing mob file 7404e7668f405a8889810821ba8518dd/A: appending metadata at 1733616965410 (+6 ms)Flushing mob file 7404e7668f405a8889810821ba8518dd/A: closing flushed file at 1733616965410Flushing 7404e7668f405a8889810821ba8518dd/A: appending metadata at 1733616965820 (+410 ms)Flushing 7404e7668f405a8889810821ba8518dd/A: closing flushed file at 1733616965820Flushing 7404e7668f405a8889810821ba8518dd/B: creating writer at 1733616966228 (+408 ms)Flushing 7404e7668f405a8889810821ba8518dd/B: appending metadata at 1733616966231 (+3 ms)Flushing 7404e7668f405a8889810821ba8518dd/B: closing flushed file at 1733616966231Flushing 7404e7668f405a8889810821ba8518dd/C: creating writer at 1733616966639 (+408 ms)Flushing 7404e7668f405a8889810821ba8518dd/C: appending metadata at 1733616966641 (+2 ms)Flushing 7404e7668f405a8889810821ba8518dd/C: closing flushed file at 1733616966641Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@289a586d: reopening flushed file at 1733616967057 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d6b550: reopening flushed file at 1733616967061 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dbc1954: reopening flushed file at 1733616967066 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 7404e7668f405a8889810821ba8518dd in 1670ms, sequenceid=786, compaction requested=false at 1733616967071 (+5 ms)Writing region close event to WAL at 1733616967148 (+77 ms)Running coprocessor post-close hooks at 1733616967150 (+2 ms)Closed at 1733616967150 2024-12-08T00:16:07,152 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,152 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=7404e7668f405a8889810821ba8518dd, regionState=CLOSED 2024-12-08T00:16:07,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:16:07,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-08T00:16:07,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 7404e7668f405a8889810821ba8518dd, server=c1a8cf8d4cfa,36021,1733616847970 in 1.9060 sec 2024-12-08T00:16:07,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-08T00:16:07,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7404e7668f405a8889810821ba8518dd, UNASSIGN in 1.9110 sec 2024-12-08T00:16:07,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-08T00:16:07,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9140 sec 2024-12-08T00:16:07,159 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616967158"}]},"ts":"1733616967158"} 2024-12-08T00:16:07,160 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T00:16:07,160 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T00:16:07,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9260 sec 2024-12-08T00:16:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T00:16:07,382 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:07,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T00:16:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:07,384 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T00:16:07,386 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:07,388 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,391 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/recovered.edits] 2024-12-08T00:16:07,394 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0b390d4e7dd045c49da9723428184292 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/0b390d4e7dd045c49da9723428184292 2024-12-08T00:16:07,396 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/46ef837aa09846d0ac35f1d6b492450b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/A/46ef837aa09846d0ac35f1d6b492450b 2024-12-08T00:16:07,398 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/a6f9876aeddf4b118736ea490dc8f7d0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/a6f9876aeddf4b118736ea490dc8f7d0 2024-12-08T00:16:07,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/acae3bb9798444e3999efd252d3c79b3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/B/acae3bb9798444e3999efd252d3c79b3 2024-12-08T00:16:07,403 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/376ccd57030b4e9bba0b56c4e4ec4312 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/376ccd57030b4e9bba0b56c4e4ec4312 2024-12-08T00:16:07,404 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/8e3191bc2be5488d89bc203e710351eb to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/C/8e3191bc2be5488d89bc203e710351eb 2024-12-08T00:16:07,407 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/recovered.edits/789.seqid to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd/recovered.edits/789.seqid 2024-12-08T00:16:07,408 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,408 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T00:16:07,408 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T00:16:07,409 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T00:16:07,411 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208030ec80ec97942e59b303a6be298fec8_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208030ec80ec97942e59b303a6be298fec8_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,412 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080aa55936f2ff4e0a8f2fe699a412708b_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080aa55936f2ff4e0a8f2fe699a412708b_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,413 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081cc31e81dbc1480a8f67c719c67b0b89_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081cc31e81dbc1480a8f67c719c67b0b89_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,414 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081dd5eff3ea9440ac87fa44bc0b719ba8_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081dd5eff3ea9440ac87fa44bc0b719ba8_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,415 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120826ca9c00d883488a888d7239b1913354_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120826ca9c00d883488a888d7239b1913354_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,416 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208297af6670cca4754bca7a8bb20fe2b4e_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208297af6670cca4754bca7a8bb20fe2b4e_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,417 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083abb04bbbbda4b3e91b067718790950c_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083abb04bbbbda4b3e91b067718790950c_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,418 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120841c40de5545c4f23945f5d5595634fac_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120841c40de5545c4f23945f5d5595634fac_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,418 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208577854ba23434f1db04681af550dbb24_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208577854ba23434f1db04681af550dbb24_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,419 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208778ee66bf07a4a1fb51d89b6b865e5c3_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208778ee66bf07a4a1fb51d89b6b865e5c3_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,420 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087a7fe9e363774c29af5aa56817bc2677_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087a7fe9e363774c29af5aa56817bc2677_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,421 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087e16f16754904a5297b54fef428247f0_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087e16f16754904a5297b54fef428247f0_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,422 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120893b65cb3254e4f33b2ef9f8b501d7f49_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120893b65cb3254e4f33b2ef9f8b501d7f49_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,423 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089a4f3f303e204f2b91a514cd1fea9c0c_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089a4f3f303e204f2b91a514cd1fea9c0c_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,423 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ae18297ea7154f89a40087b1fb2c0b98_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ae18297ea7154f89a40087b1fb2c0b98_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,424 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ba2efa8a176a4d3abd737dcfd6e48329_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ba2efa8a176a4d3abd737dcfd6e48329_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,425 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bf46f5383cd6461dafbb0c408ebc01ae_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bf46f5383cd6461dafbb0c408ebc01ae_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,426 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c2b32599258c40bda19e3d8acc6c4b37_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c2b32599258c40bda19e3d8acc6c4b37_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,427 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e6fd6374256f4ee59297e5b8e1ac8b7a_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e6fd6374256f4ee59297e5b8e1ac8b7a_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,427 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f031cc22cfa04bfba36361125e6e1536_7404e7668f405a8889810821ba8518dd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f031cc22cfa04bfba36361125e6e1536_7404e7668f405a8889810821ba8518dd 2024-12-08T00:16:07,428 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T00:16:07,429 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:07,431 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T00:16:07,432 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T00:16:07,433 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:07,433 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T00:16:07,433 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733616967433"}]},"ts":"9223372036854775807"} 2024-12-08T00:16:07,435 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T00:16:07,435 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7404e7668f405a8889810821ba8518dd, NAME => 'TestAcidGuarantees,,1733616937822.7404e7668f405a8889810821ba8518dd.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T00:16:07,435 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T00:16:07,435 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733616967435"}]},"ts":"9223372036854775807"} 2024-12-08T00:16:07,436 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T00:16:07,437 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:07,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-12-08T00:16:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T00:16:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T00:16:07,494 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:07,501 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1975563135_22 at /127.0.0.1:55514 [Waiting for operation #323] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1076213060_22 at /127.0.0.1:44100 [Waiting for operation #793] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1076213060_22 at /127.0.0.1:55512 [Waiting for operation #319] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1975563135_22 at /127.0.0.1:57546 [Waiting for operation #701] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=429 (was 462), ProcessCount=11 (was 11), AvailableMemoryMB=9066 (was 8697) - AvailableMemoryMB LEAK? - 2024-12-08T00:16:07,511 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=429, ProcessCount=11, AvailableMemoryMB=9066 2024-12-08T00:16:07,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T00:16:07,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:16:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:07,514 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T00:16:07,514 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:07,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-08T00:16:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-08T00:16:07,516 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T00:16:07,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742231_1407 (size=1061) 2024-12-08T00:16:07,575 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T00:16:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-08T00:16:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-08T00:16:07,925 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329 2024-12-08T00:16:07,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742232_1408 (size=53) 2024-12-08T00:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-08T00:16:08,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:16:08,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c03237de284ce3f116a5db8c2dd8a9d8, disabling compactions & flushes 2024-12-08T00:16:08,332 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:08,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:08,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. after waiting 0 ms 2024-12-08T00:16:08,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:08,332 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:08,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c03237de284ce3f116a5db8c2dd8a9d8: Waiting for close lock at 1733616968332Disabling compacts and flushes for region at 1733616968332Disabling writes for close at 1733616968332Writing region close event to WAL at 1733616968332Closed at 1733616968332 2024-12-08T00:16:08,333 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T00:16:08,334 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733616968333"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733616968333"}]},"ts":"1733616968333"} 2024-12-08T00:16:08,335 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T00:16:08,336 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T00:16:08,336 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616968336"}]},"ts":"1733616968336"} 2024-12-08T00:16:08,338 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T00:16:08,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c03237de284ce3f116a5db8c2dd8a9d8, ASSIGN}] 2024-12-08T00:16:08,339 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c03237de284ce3f116a5db8c2dd8a9d8, ASSIGN 2024-12-08T00:16:08,340 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c03237de284ce3f116a5db8c2dd8a9d8, ASSIGN; state=OFFLINE, location=c1a8cf8d4cfa,36021,1733616847970; forceNewPlan=false, retain=false 2024-12-08T00:16:08,491 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=c03237de284ce3f116a5db8c2dd8a9d8, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:08,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c03237de284ce3f116a5db8c2dd8a9d8, ASSIGN because future has completed 2024-12-08T00:16:08,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:16:08,647 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:08,647 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:16:08,648 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,648 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:16:08,648 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,648 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,650 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-08T00:16:08,652 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:16:08,652 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c03237de284ce3f116a5db8c2dd8a9d8 columnFamilyName A 2024-12-08T00:16:08,652 DEBUG [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:08,652 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] regionserver.HStore(327): Store=c03237de284ce3f116a5db8c2dd8a9d8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:16:08,653 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,654 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:16:08,654 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c03237de284ce3f116a5db8c2dd8a9d8 columnFamilyName B 2024-12-08T00:16:08,654 DEBUG [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:08,654 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] regionserver.HStore(327): Store=c03237de284ce3f116a5db8c2dd8a9d8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:16:08,655 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,656 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:16:08,656 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c03237de284ce3f116a5db8c2dd8a9d8 columnFamilyName C 2024-12-08T00:16:08,656 DEBUG [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:08,656 INFO [StoreOpener-c03237de284ce3f116a5db8c2dd8a9d8-1 {}] regionserver.HStore(327): Store=c03237de284ce3f116a5db8c2dd8a9d8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:16:08,656 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:08,657 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,657 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,657 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,658 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,658 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,659 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:16:08,659 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,661 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T00:16:08,661 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened c03237de284ce3f116a5db8c2dd8a9d8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67000843, jitterRate=-0.0016096383333206177}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:16:08,662 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:08,662 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for c03237de284ce3f116a5db8c2dd8a9d8: Running coprocessor pre-open hook at 1733616968648Writing region info on filesystem at 1733616968648Initializing all the Stores at 1733616968649 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616968649Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616968650 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616968650Cleaning up temporary data from old regions at 1733616968658 (+8 ms)Running coprocessor post-open hooks at 1733616968662 (+4 ms)Region opened successfully at 1733616968662 2024-12-08T00:16:08,663 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., pid=114, masterSystemTime=1733616968644 2024-12-08T00:16:08,665 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:08,665 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:08,665 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=c03237de284ce3f116a5db8c2dd8a9d8, regionState=OPEN, openSeqNum=2, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:08,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:16:08,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-08T00:16:08,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 in 175 msec 2024-12-08T00:16:08,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-08T00:16:08,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c03237de284ce3f116a5db8c2dd8a9d8, ASSIGN in 331 msec 2024-12-08T00:16:08,672 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T00:16:08,672 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616968672"}]},"ts":"1733616968672"} 2024-12-08T00:16:08,673 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T00:16:08,674 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T00:16:08,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-12-08T00:16:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-08T00:16:09,663 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:09,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,668 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@184b0249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,671 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9f2dd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,672 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T00:16:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:16:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5210f803 2024-12-08T00:16:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T00:16:09,676 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T00:16:09,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,679 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c141cc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,680 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dcde5ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,682 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ba07f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,683 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3477d155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,685 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f6526b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,686 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@498b693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,688 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4580235e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,689 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b813b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,690 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683ac21c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,691 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cfc091, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,693 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f6933d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,694 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24e8ad88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,695 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5374e7f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,696 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39aeab91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,698 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c334e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,699 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69fd9bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,701 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6912e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,702 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29109312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:09,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:09,704 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4d0dd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:09,705 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5682fc1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:09,706 DEBUG [Thread-1847 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,706 DEBUG [Thread-1864 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,706 DEBUG [Thread-1849 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,706 DEBUG [Thread-1856 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,706 DEBUG [Thread-1862 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,707 DEBUG [Thread-1853 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,707 DEBUG [Thread-1851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,708 DEBUG [Thread-1860 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,708 DEBUG [Thread-1858 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,710 DEBUG [Thread-1845 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:09,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:09,711 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:09,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-08T00:16:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T00:16:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,717 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2] 2024-12-08T00:16:09,719 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:09,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:09,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:16:09,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:09,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:09,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:09,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:09,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:09,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:09,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/c7eddb00b48947889a28084d24f959eb is 50, key is test_row_0/A:col10/1733616969720/Put/seqid=0 2024-12-08T00:16:09,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733616969730, max timestamp=1733616969744Num uniques 859; 2024-12-08T00:16:09,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T00:16:09,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733616969730, max timestamp=1733616969744Num uniques 859; 2024-12-08T00:16:09,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T00:16:09,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733616969730, max timestamp=1733616969744Num uniques 859; 2024-12-08T00:16:09,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T00:16:09,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742233_1409 (size=11997) 2024-12-08T00:16:09,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/c7eddb00b48947889a28084d24f959eb 2024-12-08T00:16:09,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/c8ca2b4a2cad4576916bb585980cff55 is 50, key is test_row_0/B:col10/1733616969720/Put/seqid=0 2024-12-08T00:16:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617029764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617029767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617029768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:09,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617029769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617029770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:09,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742234_1410 (size=9657) 2024-12-08T00:16:09,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/c8ca2b4a2cad4576916bb585980cff55 2024-12-08T00:16:09,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/0c86dc442a60424c8f3f57fa556babe4 is 50, key is test_row_0/C:col10/1733616969720/Put/seqid=0 2024-12-08T00:16:09,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742235_1411 (size=9657) 2024-12-08T00:16:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T00:16:09,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-08T00:16:09,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:09,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:09,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:09,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:09,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:09,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617029881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:09,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617029881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617029881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:09,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617029881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617029883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-08T00:16:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:10,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:10,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:10,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:10,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:10,029 WARN [regionserver/c1a8cf8d4cfa:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-12-08T00:16:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T00:16:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617030091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617030101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617030101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617030101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617030102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-08T00:16:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:10,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:10,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/0c86dc442a60424c8f3f57fa556babe4 2024-12-08T00:16:10,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/c7eddb00b48947889a28084d24f959eb as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c7eddb00b48947889a28084d24f959eb 2024-12-08T00:16:10,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c7eddb00b48947889a28084d24f959eb, entries=150, sequenceid=11, filesize=11.7 K 2024-12-08T00:16:10,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/c8ca2b4a2cad4576916bb585980cff55 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/c8ca2b4a2cad4576916bb585980cff55 2024-12-08T00:16:10,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/c8ca2b4a2cad4576916bb585980cff55, entries=100, sequenceid=11, filesize=9.4 K 2024-12-08T00:16:10,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/0c86dc442a60424c8f3f57fa556babe4 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0c86dc442a60424c8f3f57fa556babe4 2024-12-08T00:16:10,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0c86dc442a60424c8f3f57fa556babe4, entries=100, sequenceid=11, filesize=9.4 K 2024-12-08T00:16:10,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c03237de284ce3f116a5db8c2dd8a9d8 in 507ms, sequenceid=11, compaction requested=false 2024-12-08T00:16:10,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:10,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-08T00:16:10,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:10,331 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-08T00:16:10,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:10,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:10,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:10,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:10,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:10,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:10,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/10df33ec03cb4d1b9539a3c68fd16987 is 50, key is test_row_0/A:col10/1733616969769/Put/seqid=0 2024-12-08T00:16:10,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742236_1412 (size=12001) 2024-12-08T00:16:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T00:16:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:10,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:10,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617030427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617030429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617030430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617030431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617030431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617030541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617030541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617030541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617030550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617030550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,739 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/10df33ec03cb4d1b9539a3c68fd16987 2024-12-08T00:16:10,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/4178b8f0a57e40fabc6ca04e7372d842 is 50, key is test_row_0/B:col10/1733616969769/Put/seqid=0 2024-12-08T00:16:10,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742237_1413 (size=12001) 2024-12-08T00:16:10,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617030751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617030751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617030761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617030762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617030762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T00:16:11,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617031061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617031062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617031071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617031071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617031071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/4178b8f0a57e40fabc6ca04e7372d842 2024-12-08T00:16:11,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/685218372f714bd8bf6ad3d22ba381d5 is 50, key is test_row_0/C:col10/1733616969769/Put/seqid=0 2024-12-08T00:16:11,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742238_1414 (size=12001) 2024-12-08T00:16:11,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/685218372f714bd8bf6ad3d22ba381d5 2024-12-08T00:16:11,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/10df33ec03cb4d1b9539a3c68fd16987 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/10df33ec03cb4d1b9539a3c68fd16987 2024-12-08T00:16:11,169 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/10df33ec03cb4d1b9539a3c68fd16987, entries=150, sequenceid=46, filesize=11.7 K 2024-12-08T00:16:11,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/4178b8f0a57e40fabc6ca04e7372d842 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/4178b8f0a57e40fabc6ca04e7372d842 2024-12-08T00:16:11,177 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/4178b8f0a57e40fabc6ca04e7372d842, entries=150, sequenceid=46, filesize=11.7 K 2024-12-08T00:16:11,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/685218372f714bd8bf6ad3d22ba381d5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/685218372f714bd8bf6ad3d22ba381d5 2024-12-08T00:16:11,181 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/685218372f714bd8bf6ad3d22ba381d5, entries=150, sequenceid=46, filesize=11.7 K 2024-12-08T00:16:11,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for c03237de284ce3f116a5db8c2dd8a9d8 in 850ms, sequenceid=46, compaction requested=false 2024-12-08T00:16:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-08T00:16:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-08T00:16:11,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-08T00:16:11,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4640 sec 2024-12-08T00:16:11,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4740 sec 2024-12-08T00:16:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:11,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:16:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:11,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/7d9874938c9b4650b334ed0fd052ce9d is 50, key is test_row_0/A:col10/1733616971585/Put/seqid=0 2024-12-08T00:16:11,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742239_1415 (size=14341) 2024-12-08T00:16:11,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617031606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617031613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617031614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617031615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617031616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617031721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617031731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617031731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617031731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617031731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T00:16:11,861 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:11,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-08T00:16:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T00:16:11,863 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:11,864 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:11,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:11,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617031940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617031940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617031941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617031941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617031941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T00:16:12,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/7d9874938c9b4650b334ed0fd052ce9d 2024-12-08T00:16:12,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/911436bf40164f82b7b9f2c9d92d7305 is 50, key is test_row_0/B:col10/1733616971585/Put/seqid=0 2024-12-08T00:16:12,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T00:16:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742240_1416 (size=12001) 2024-12-08T00:16:12,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T00:16:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T00:16:12,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617032251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617032251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617032251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617032251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617032252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T00:16:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/911436bf40164f82b7b9f2c9d92d7305 2024-12-08T00:16:12,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/439f4c11f7e941968f1ae82bb7db5404 is 50, key is test_row_0/C:col10/1733616971585/Put/seqid=0 2024-12-08T00:16:12,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742241_1417 (size=12001) 2024-12-08T00:16:12,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T00:16:12,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:12,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T00:16:12,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T00:16:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,669 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T00:16:12,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617032761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617032761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617032762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617032762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617032762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:12,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T00:16:12,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:12,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:12,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/439f4c11f7e941968f1ae82bb7db5404 2024-12-08T00:16:12,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/7d9874938c9b4650b334ed0fd052ce9d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/7d9874938c9b4650b334ed0fd052ce9d 2024-12-08T00:16:12,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/7d9874938c9b4650b334ed0fd052ce9d, entries=200, sequenceid=60, filesize=14.0 K 2024-12-08T00:16:12,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/911436bf40164f82b7b9f2c9d92d7305 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/911436bf40164f82b7b9f2c9d92d7305 2024-12-08T00:16:12,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/911436bf40164f82b7b9f2c9d92d7305, entries=150, sequenceid=60, filesize=11.7 K 2024-12-08T00:16:12,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/439f4c11f7e941968f1ae82bb7db5404 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/439f4c11f7e941968f1ae82bb7db5404 2024-12-08T00:16:12,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/439f4c11f7e941968f1ae82bb7db5404, entries=150, sequenceid=60, filesize=11.7 K 2024-12-08T00:16:12,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c03237de284ce3f116a5db8c2dd8a9d8 in 1263ms, sequenceid=60, compaction requested=true 2024-12-08T00:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:12,848 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:12,848 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:12,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:12,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:12,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/A is initiating minor compaction (all files) 2024-12-08T00:16:12,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/B is initiating minor compaction (all files) 2024-12-08T00:16:12,849 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/A in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,849 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/B in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,849 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c7eddb00b48947889a28084d24f959eb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/10df33ec03cb4d1b9539a3c68fd16987, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/7d9874938c9b4650b334ed0fd052ce9d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=37.4 K 2024-12-08T00:16:12,849 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/c8ca2b4a2cad4576916bb585980cff55, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/4178b8f0a57e40fabc6ca04e7372d842, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/911436bf40164f82b7b9f2c9d92d7305] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=32.9 K 2024-12-08T00:16:12,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7eddb00b48947889a28084d24f959eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733616969716 2024-12-08T00:16:12,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ca2b4a2cad4576916bb585980cff55, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733616969716 2024-12-08T00:16:12,850 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 4178b8f0a57e40fabc6ca04e7372d842, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733616969767 2024-12-08T00:16:12,850 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10df33ec03cb4d1b9539a3c68fd16987, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733616969767 2024-12-08T00:16:12,850 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 911436bf40164f82b7b9f2c9d92d7305, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733616970426 2024-12-08T00:16:12,850 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d9874938c9b4650b334ed0fd052ce9d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733616970426 2024-12-08T00:16:12,857 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:12,857 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#A#compaction#343 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:12,857 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/8254e5e9e3c740be97a773902748b9fc is 50, key is test_row_0/B:col10/1733616971585/Put/seqid=0 2024-12-08T00:16:12,857 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/bf807e8caae14c8d83b54c4248ab1fb9 is 50, key is test_row_0/A:col10/1733616971585/Put/seqid=0 2024-12-08T00:16:12,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742243_1419 (size=12104) 2024-12-08T00:16:12,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742242_1418 (size=12104) 2024-12-08T00:16:12,869 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/8254e5e9e3c740be97a773902748b9fc as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8254e5e9e3c740be97a773902748b9fc 2024-12-08T00:16:12,874 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/B of c03237de284ce3f116a5db8c2dd8a9d8 into 8254e5e9e3c740be97a773902748b9fc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:12,874 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:12,874 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/B, priority=13, startTime=1733616972848; duration=0sec 2024-12-08T00:16:12,874 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:12,874 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:B 2024-12-08T00:16:12,874 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:12,877 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:12,877 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/C is initiating minor compaction (all files) 2024-12-08T00:16:12,877 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/C in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,877 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0c86dc442a60424c8f3f57fa556babe4, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/685218372f714bd8bf6ad3d22ba381d5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/439f4c11f7e941968f1ae82bb7db5404] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=32.9 K 2024-12-08T00:16:12,877 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c86dc442a60424c8f3f57fa556babe4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733616969716 2024-12-08T00:16:12,877 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 685218372f714bd8bf6ad3d22ba381d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733616969767 2024-12-08T00:16:12,878 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 439f4c11f7e941968f1ae82bb7db5404, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733616970426 2024-12-08T00:16:12,884 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#C#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:12,885 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/5724c7c6a9454cea86b29376d8872b6f is 50, key is test_row_0/C:col10/1733616971585/Put/seqid=0 2024-12-08T00:16:12,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742244_1420 (size=12104) 2024-12-08T00:16:12,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T00:16:12,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:12,934 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T00:16:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:12,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/12e179f29d5d4f1d851f033627bd181e is 50, key is test_row_0/A:col10/1733616971615/Put/seqid=0 2024-12-08T00:16:12,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742245_1421 (size=12001) 2024-12-08T00:16:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T00:16:13,267 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/bf807e8caae14c8d83b54c4248ab1fb9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/bf807e8caae14c8d83b54c4248ab1fb9 2024-12-08T00:16:13,271 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/A of c03237de284ce3f116a5db8c2dd8a9d8 into bf807e8caae14c8d83b54c4248ab1fb9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:13,271 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:13,271 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/A, priority=13, startTime=1733616972848; duration=0sec 2024-12-08T00:16:13,271 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:13,271 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:A 2024-12-08T00:16:13,303 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/5724c7c6a9454cea86b29376d8872b6f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/5724c7c6a9454cea86b29376d8872b6f 2024-12-08T00:16:13,307 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/C of c03237de284ce3f116a5db8c2dd8a9d8 into 5724c7c6a9454cea86b29376d8872b6f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:13,307 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:13,307 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/C, priority=13, startTime=1733616972848; duration=0sec 2024-12-08T00:16:13,307 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:13,307 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:C 2024-12-08T00:16:13,343 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/12e179f29d5d4f1d851f033627bd181e 2024-12-08T00:16:13,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/2573ac9ed505472b93a726abda9d9e54 is 50, key is test_row_0/B:col10/1733616971615/Put/seqid=0 2024-12-08T00:16:13,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742246_1422 (size=12001) 2024-12-08T00:16:13,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/2573ac9ed505472b93a726abda9d9e54 2024-12-08T00:16:13,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/0facd1babdcf4170959056e375a99b8d is 50, key is test_row_0/C:col10/1733616971615/Put/seqid=0 2024-12-08T00:16:13,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742247_1423 (size=12001) 2024-12-08T00:16:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:13,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:13,813 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/0facd1babdcf4170959056e375a99b8d 2024-12-08T00:16:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/12e179f29d5d4f1d851f033627bd181e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/12e179f29d5d4f1d851f033627bd181e 2024-12-08T00:16:13,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617033807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617033807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:13,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617033808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617033808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:13,820 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/12e179f29d5d4f1d851f033627bd181e, entries=150, sequenceid=79, filesize=11.7 K 2024-12-08T00:16:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/2573ac9ed505472b93a726abda9d9e54 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2573ac9ed505472b93a726abda9d9e54 2024-12-08T00:16:13,824 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2573ac9ed505472b93a726abda9d9e54, entries=150, sequenceid=79, filesize=11.7 K 2024-12-08T00:16:13,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/0facd1babdcf4170959056e375a99b8d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0facd1babdcf4170959056e375a99b8d 2024-12-08T00:16:13,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617033818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0facd1babdcf4170959056e375a99b8d, entries=150, sequenceid=79, filesize=11.7 K 2024-12-08T00:16:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:13,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c03237de284ce3f116a5db8c2dd8a9d8 in 894ms, sequenceid=79, compaction requested=false 2024-12-08T00:16:13,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:13,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:13,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-08T00:16:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-08T00:16:13,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-08T00:16:13,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9670 sec 2024-12-08T00:16:13,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.9710 sec 2024-12-08T00:16:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:13,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T00:16:13,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:13,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:13,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:13,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:13,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:13,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:13,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/a82a631a13e94be38df81fb628087bf1 is 50, key is test_row_0/A:col10/1733616973778/Put/seqid=0 2024-12-08T00:16:13,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742248_1424 (size=14341) 2024-12-08T00:16:13,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617033946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617033947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617033951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617033952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617033952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T00:16:14,011 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:14,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-08T00:16:14,013 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T00:16:14,014 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:14,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:14,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617034061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617034061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617034071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617034071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617034071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T00:16:14,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T00:16:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617034271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617034271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617034281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617034281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617034281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T00:16:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T00:16:14,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/a82a631a13e94be38df81fb628087bf1 2024-12-08T00:16:14,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/04ed9ed9e4df4d6a967edeabdf94143c is 50, key is test_row_0/B:col10/1733616973778/Put/seqid=0 2024-12-08T00:16:14,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742249_1425 (size=12001) 2024-12-08T00:16:14,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T00:16:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617034580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617034591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617034591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617034591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617034591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:14,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T00:16:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T00:16:14,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/04ed9ed9e4df4d6a967edeabdf94143c 2024-12-08T00:16:14,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/91e4337e214b4292a6f73ed2270678db is 50, key is test_row_0/C:col10/1733616973778/Put/seqid=0 2024-12-08T00:16:14,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742250_1426 (size=12001) 2024-12-08T00:16:14,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T00:16:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T00:16:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:14,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:15,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T00:16:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:15,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:15,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617035100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:15,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617035111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:15,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617035111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617035111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617035111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T00:16:15,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/91e4337e214b4292a6f73ed2270678db 2024-12-08T00:16:15,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/a82a631a13e94be38df81fb628087bf1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a82a631a13e94be38df81fb628087bf1 2024-12-08T00:16:15,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a82a631a13e94be38df81fb628087bf1, entries=200, sequenceid=96, filesize=14.0 K 2024-12-08T00:16:15,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/04ed9ed9e4df4d6a967edeabdf94143c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/04ed9ed9e4df4d6a967edeabdf94143c 2024-12-08T00:16:15,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/04ed9ed9e4df4d6a967edeabdf94143c, entries=150, sequenceid=96, filesize=11.7 K 2024-12-08T00:16:15,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/91e4337e214b4292a6f73ed2270678db as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/91e4337e214b4292a6f73ed2270678db 2024-12-08T00:16:15,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/91e4337e214b4292a6f73ed2270678db, entries=150, sequenceid=96, filesize=11.7 K 2024-12-08T00:16:15,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c03237de284ce3f116a5db8c2dd8a9d8 in 1264ms, sequenceid=96, compaction requested=true 2024-12-08T00:16:15,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:15,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:15,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:15,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:15,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:15,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:15,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:15,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:15,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:15,193 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:15,193 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/A is initiating minor compaction (all files) 2024-12-08T00:16:15,193 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/A in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:15,193 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/bf807e8caae14c8d83b54c4248ab1fb9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/12e179f29d5d4f1d851f033627bd181e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a82a631a13e94be38df81fb628087bf1] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=37.5 K 2024-12-08T00:16:15,194 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:15,194 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/B is initiating minor compaction (all files) 2024-12-08T00:16:15,194 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/B in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:15,194 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8254e5e9e3c740be97a773902748b9fc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2573ac9ed505472b93a726abda9d9e54, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/04ed9ed9e4df4d6a967edeabdf94143c] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=35.3 K 2024-12-08T00:16:15,194 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting bf807e8caae14c8d83b54c4248ab1fb9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733616970426 2024-12-08T00:16:15,195 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e179f29d5d4f1d851f033627bd181e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733616971603 2024-12-08T00:16:15,195 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8254e5e9e3c740be97a773902748b9fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733616970426 2024-12-08T00:16:15,196 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2573ac9ed505472b93a726abda9d9e54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733616971603 2024-12-08T00:16:15,196 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a82a631a13e94be38df81fb628087bf1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733616973778 2024-12-08T00:16:15,196 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ed9ed9e4df4d6a967edeabdf94143c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733616973778 2024-12-08T00:16:15,206 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#B#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:15,206 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#A#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:15,207 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/6674d50440a242bab29553a6ea600c37 is 50, key is test_row_0/B:col10/1733616973778/Put/seqid=0 2024-12-08T00:16:15,207 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/68c6fd49ef534be59b76fdd182f5c58f is 50, key is test_row_0/A:col10/1733616973778/Put/seqid=0 2024-12-08T00:16:15,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742251_1427 (size=12207) 2024-12-08T00:16:15,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742252_1428 (size=12207) 2024-12-08T00:16:15,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T00:16:15,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:15,242 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T00:16:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/81e5c1b902ab4d5a9fcff4e62f034eb7 is 50, key is test_row_0/A:col10/1733616973950/Put/seqid=0 2024-12-08T00:16:15,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742253_1429 (size=12001) 2024-12-08T00:16:15,622 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/6674d50440a242bab29553a6ea600c37 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6674d50440a242bab29553a6ea600c37 2024-12-08T00:16:15,625 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/B of c03237de284ce3f116a5db8c2dd8a9d8 into 6674d50440a242bab29553a6ea600c37(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:15,625 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:15,625 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/B, priority=13, startTime=1733616975188; duration=0sec 2024-12-08T00:16:15,625 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:15,625 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:B 2024-12-08T00:16:15,625 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:15,626 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:15,626 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/C is initiating minor compaction (all files) 2024-12-08T00:16:15,626 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/C in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:15,626 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/5724c7c6a9454cea86b29376d8872b6f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0facd1babdcf4170959056e375a99b8d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/91e4337e214b4292a6f73ed2270678db] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=35.3 K 2024-12-08T00:16:15,628 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5724c7c6a9454cea86b29376d8872b6f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733616970426 2024-12-08T00:16:15,628 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0facd1babdcf4170959056e375a99b8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733616971603 2024-12-08T00:16:15,628 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91e4337e214b4292a6f73ed2270678db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733616973778 2024-12-08T00:16:15,631 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/68c6fd49ef534be59b76fdd182f5c58f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/68c6fd49ef534be59b76fdd182f5c58f 2024-12-08T00:16:15,634 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#C#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:15,635 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/e11b555cd22142ae90bfdcb39de5ea0f is 50, key is test_row_0/C:col10/1733616973778/Put/seqid=0 2024-12-08T00:16:15,635 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/A of c03237de284ce3f116a5db8c2dd8a9d8 into 68c6fd49ef534be59b76fdd182f5c58f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:15,635 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:15,635 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/A, priority=13, startTime=1733616975188; duration=0sec 2024-12-08T00:16:15,635 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:15,635 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:A 2024-12-08T00:16:15,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742254_1430 (size=12207) 2024-12-08T00:16:15,667 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/81e5c1b902ab4d5a9fcff4e62f034eb7 2024-12-08T00:16:15,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/b08767fc5f444c9eb52634ddd8796998 is 50, key is test_row_0/B:col10/1733616973950/Put/seqid=0 2024-12-08T00:16:15,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742255_1431 (size=12001) 2024-12-08T00:16:16,046 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/e11b555cd22142ae90bfdcb39de5ea0f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e11b555cd22142ae90bfdcb39de5ea0f 2024-12-08T00:16:16,049 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/C of c03237de284ce3f116a5db8c2dd8a9d8 into e11b555cd22142ae90bfdcb39de5ea0f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:16,049 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:16,050 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/C, priority=13, startTime=1733616975188; duration=0sec 2024-12-08T00:16:16,050 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:16,050 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:C 2024-12-08T00:16:16,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/b08767fc5f444c9eb52634ddd8796998 2024-12-08T00:16:16,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/88f6c014ad8b41cfa925085abd4e555c is 50, key is test_row_0/C:col10/1733616973950/Put/seqid=0 2024-12-08T00:16:16,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742256_1432 (size=12001) 2024-12-08T00:16:16,093 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/88f6c014ad8b41cfa925085abd4e555c 2024-12-08T00:16:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/81e5c1b902ab4d5a9fcff4e62f034eb7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81e5c1b902ab4d5a9fcff4e62f034eb7 2024-12-08T00:16:16,108 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81e5c1b902ab4d5a9fcff4e62f034eb7, entries=150, sequenceid=114, filesize=11.7 K 2024-12-08T00:16:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/b08767fc5f444c9eb52634ddd8796998 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b08767fc5f444c9eb52634ddd8796998 2024-12-08T00:16:16,122 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b08767fc5f444c9eb52634ddd8796998, entries=150, sequenceid=114, filesize=11.7 K 2024-12-08T00:16:16,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/88f6c014ad8b41cfa925085abd4e555c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88f6c014ad8b41cfa925085abd4e555c 2024-12-08T00:16:16,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:16,130 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88f6c014ad8b41cfa925085abd4e555c, entries=150, sequenceid=114, filesize=11.7 K 2024-12-08T00:16:16,132 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for c03237de284ce3f116a5db8c2dd8a9d8 in 891ms, sequenceid=114, compaction requested=false 2024-12-08T00:16:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-08T00:16:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-08T00:16:16,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-08T00:16:16,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-12-08T00:16:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:16,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:16:16,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:16,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:16,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:16,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:16,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:16,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:16,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.1250 sec 2024-12-08T00:16:16,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/df327077af6442e8861d88047e00edbe is 50, key is test_row_0/A:col10/1733616976131/Put/seqid=0 2024-12-08T00:16:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742257_1433 (size=14341) 2024-12-08T00:16:16,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/df327077af6442e8861d88047e00edbe 2024-12-08T00:16:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T00:16:16,162 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:16,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-08T00:16:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T00:16:16,165 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:16,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/fe0739da1720444b8225857bb40accaa is 50, key is test_row_0/B:col10/1733616976131/Put/seqid=0 2024-12-08T00:16:16,165 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:16,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:16,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742258_1434 (size=12001) 2024-12-08T00:16:16,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/fe0739da1720444b8225857bb40accaa 2024-12-08T00:16:16,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/9b0d50a7b3f8465fb0c128eb0cdc65ac is 50, key is test_row_0/C:col10/1733616976131/Put/seqid=0 2024-12-08T00:16:16,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617036175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616976166, max timestamp=1733616976185Num uniques 859; 2024-12-08T00:16:16,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616976166, max timestamp=1733616976185Num uniques 859; 2024-12-08T00:16:16,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:16,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:16,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733616976166, max timestamp=1733616976185Num uniques 859; 2024-12-08T00:16:16,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742259_1435 (size=12001) 2024-12-08T00:16:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617036221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617036222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617036228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617036229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T00:16:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617036291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T00:16:16,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:16,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:16,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:16,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:16,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:16,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617036341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617036341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617036341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617036350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T00:16:16,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:16,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:16,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:16,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:16,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T00:16:16,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617036501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617036551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617036551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617036551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617036561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/9b0d50a7b3f8465fb0c128eb0cdc65ac 2024-12-08T00:16:16,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/df327077af6442e8861d88047e00edbe as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/df327077af6442e8861d88047e00edbe 2024-12-08T00:16:16,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/df327077af6442e8861d88047e00edbe, entries=200, sequenceid=128, filesize=14.0 K 2024-12-08T00:16:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/fe0739da1720444b8225857bb40accaa as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/fe0739da1720444b8225857bb40accaa 2024-12-08T00:16:16,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/fe0739da1720444b8225857bb40accaa, entries=150, sequenceid=128, filesize=11.7 K 2024-12-08T00:16:16,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/9b0d50a7b3f8465fb0c128eb0cdc65ac as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/9b0d50a7b3f8465fb0c128eb0cdc65ac 2024-12-08T00:16:16,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/9b0d50a7b3f8465fb0c128eb0cdc65ac, entries=150, sequenceid=128, filesize=11.7 K 2024-12-08T00:16:16,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c03237de284ce3f116a5db8c2dd8a9d8 in 476ms, sequenceid=128, compaction requested=true 2024-12-08T00:16:16,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:16,613 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:16,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:16,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:16,613 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:16,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:16,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:16,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:16,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:16,614 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:16,614 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:16,614 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/A is initiating minor compaction (all files) 2024-12-08T00:16:16,614 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/B is initiating minor compaction (all files) 2024-12-08T00:16:16,614 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/B in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:16,614 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/A in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:16,614 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6674d50440a242bab29553a6ea600c37, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b08767fc5f444c9eb52634ddd8796998, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/fe0739da1720444b8225857bb40accaa] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=35.4 K 2024-12-08T00:16:16,614 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/68c6fd49ef534be59b76fdd182f5c58f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81e5c1b902ab4d5a9fcff4e62f034eb7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/df327077af6442e8861d88047e00edbe] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=37.6 K 2024-12-08T00:16:16,615 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6674d50440a242bab29553a6ea600c37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733616973778 2024-12-08T00:16:16,615 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c6fd49ef534be59b76fdd182f5c58f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733616973778 2024-12-08T00:16:16,615 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting b08767fc5f444c9eb52634ddd8796998, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733616973945 2024-12-08T00:16:16,615 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81e5c1b902ab4d5a9fcff4e62f034eb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733616973945 2024-12-08T00:16:16,616 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting fe0739da1720444b8225857bb40accaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616976131 2024-12-08T00:16:16,616 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting df327077af6442e8861d88047e00edbe, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616976131 2024-12-08T00:16:16,625 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#A#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:16,625 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:16,625 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/cf2e8464bc2041f594c999f47e162561 is 50, key is test_row_0/B:col10/1733616976131/Put/seqid=0 2024-12-08T00:16:16,625 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/ca5726e99be8401f8f08af2365525009 is 50, key is test_row_0/A:col10/1733616976131/Put/seqid=0 2024-12-08T00:16:16,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T00:16:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:16,634 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-08T00:16:16,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742261_1437 (size=12309) 2024-12-08T00:16:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:16,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742260_1436 (size=12309) 2024-12-08T00:16:16,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/11b53de74c5645a2a98e40e3ca1b2f71 is 50, key is test_row_0/A:col10/1733616976227/Put/seqid=0 2024-12-08T00:16:16,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742262_1438 (size=12151) 2024-12-08T00:16:16,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/11b53de74c5645a2a98e40e3ca1b2f71 2024-12-08T00:16:16,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/241a749e0f9847198350d52ccd3cccda is 50, key is test_row_0/B:col10/1733616976227/Put/seqid=0 2024-12-08T00:16:16,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742263_1439 (size=12151) 2024-12-08T00:16:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T00:16:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:16,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:16,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617036872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617036874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617036875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617036880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617036883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617037001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617037001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617037002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617037002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,037 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/cf2e8464bc2041f594c999f47e162561 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/cf2e8464bc2041f594c999f47e162561 2024-12-08T00:16:17,045 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/B of c03237de284ce3f116a5db8c2dd8a9d8 into cf2e8464bc2041f594c999f47e162561(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:17,045 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:17,045 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/B, priority=13, startTime=1733616976613; duration=0sec 2024-12-08T00:16:17,045 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:17,045 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:B 2024-12-08T00:16:17,045 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:17,046 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:17,046 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/C is initiating minor compaction (all files) 2024-12-08T00:16:17,046 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/C in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,046 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e11b555cd22142ae90bfdcb39de5ea0f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88f6c014ad8b41cfa925085abd4e555c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/9b0d50a7b3f8465fb0c128eb0cdc65ac] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=35.4 K 2024-12-08T00:16:17,047 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e11b555cd22142ae90bfdcb39de5ea0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733616973778 2024-12-08T00:16:17,047 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f6c014ad8b41cfa925085abd4e555c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733616973945 2024-12-08T00:16:17,047 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b0d50a7b3f8465fb0c128eb0cdc65ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616976131 2024-12-08T00:16:17,049 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/ca5726e99be8401f8f08af2365525009 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ca5726e99be8401f8f08af2365525009 2024-12-08T00:16:17,053 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#C#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:17,053 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/63762f72a5ff4f5da7338f41384f5cf5 is 50, key is test_row_0/C:col10/1733616976131/Put/seqid=0 2024-12-08T00:16:17,054 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/A of c03237de284ce3f116a5db8c2dd8a9d8 into ca5726e99be8401f8f08af2365525009(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:17,054 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:17,054 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/A, priority=13, startTime=1733616976613; duration=0sec 2024-12-08T00:16:17,054 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:17,054 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:A 2024-12-08T00:16:17,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742264_1440 (size=12309) 2024-12-08T00:16:17,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/241a749e0f9847198350d52ccd3cccda 2024-12-08T00:16:17,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/f48812e699e04ed5beaacb89984a5724 is 50, key is test_row_0/C:col10/1733616976227/Put/seqid=0 2024-12-08T00:16:17,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742265_1441 (size=12151) 2024-12-08T00:16:17,115 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/f48812e699e04ed5beaacb89984a5724 2024-12-08T00:16:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/11b53de74c5645a2a98e40e3ca1b2f71 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/11b53de74c5645a2a98e40e3ca1b2f71 2024-12-08T00:16:17,122 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/11b53de74c5645a2a98e40e3ca1b2f71, entries=150, sequenceid=163, filesize=11.9 K 2024-12-08T00:16:17,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/241a749e0f9847198350d52ccd3cccda as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/241a749e0f9847198350d52ccd3cccda 2024-12-08T00:16:17,127 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/241a749e0f9847198350d52ccd3cccda, entries=150, sequenceid=163, filesize=11.9 K 2024-12-08T00:16:17,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/f48812e699e04ed5beaacb89984a5724 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/f48812e699e04ed5beaacb89984a5724 2024-12-08T00:16:17,131 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/f48812e699e04ed5beaacb89984a5724, entries=150, sequenceid=163, filesize=11.9 K 2024-12-08T00:16:17,132 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for c03237de284ce3f116a5db8c2dd8a9d8 in 498ms, sequenceid=163, compaction requested=false 2024-12-08T00:16:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-08T00:16:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-08T00:16:17,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-08T00:16:17,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 971 msec 2024-12-08T00:16:17,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 975 msec 2024-12-08T00:16:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:17,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T00:16:17,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:17,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:17,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:17,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:17,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:17,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:17,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/eaea035b51c04dd2ba3290b39c23b11d is 50, key is test_row_0/A:col10/1733616976881/Put/seqid=0 2024-12-08T00:16:17,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617037275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617037275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617037283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742266_1442 (size=12151) 2024-12-08T00:16:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T00:16:17,301 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:17,302 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-08T00:16:17,304 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T00:16:17,304 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:17,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:17,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617037293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T00:16:17,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617037401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617037401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617037401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617037401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617037421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T00:16:17,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:17,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:17,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:17,466 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/63762f72a5ff4f5da7338f41384f5cf5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/63762f72a5ff4f5da7338f41384f5cf5 2024-12-08T00:16:17,475 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/C of c03237de284ce3f116a5db8c2dd8a9d8 into 63762f72a5ff4f5da7338f41384f5cf5(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:17,475 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:17,475 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/C, priority=13, startTime=1733616976614; duration=0sec 2024-12-08T00:16:17,475 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:17,475 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:C 2024-12-08T00:16:17,576 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T00:16:17,576 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T00:16:17,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T00:16:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T00:16:17,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617037622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617037622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617037622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617037641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/eaea035b51c04dd2ba3290b39c23b11d 2024-12-08T00:16:17,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/3808e2bf214941e5a70116a2a311e9a9 is 50, key is test_row_0/B:col10/1733616976881/Put/seqid=0 2024-12-08T00:16:17,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742267_1443 (size=12151) 2024-12-08T00:16:17,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/3808e2bf214941e5a70116a2a311e9a9 2024-12-08T00:16:17,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/03a1dff10ddf4164867bc702e837488d is 50, key is test_row_0/C:col10/1733616976881/Put/seqid=0 2024-12-08T00:16:17,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742268_1444 (size=12151) 2024-12-08T00:16:17,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/03a1dff10ddf4164867bc702e837488d 2024-12-08T00:16:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/eaea035b51c04dd2ba3290b39c23b11d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/eaea035b51c04dd2ba3290b39c23b11d 2024-12-08T00:16:17,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/eaea035b51c04dd2ba3290b39c23b11d, entries=150, sequenceid=178, filesize=11.9 K 2024-12-08T00:16:17,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/3808e2bf214941e5a70116a2a311e9a9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/3808e2bf214941e5a70116a2a311e9a9 2024-12-08T00:16:17,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T00:16:17,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:17,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:17,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:17,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/3808e2bf214941e5a70116a2a311e9a9, entries=150, sequenceid=178, filesize=11.9 K 2024-12-08T00:16:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:17,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/03a1dff10ddf4164867bc702e837488d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03a1dff10ddf4164867bc702e837488d 2024-12-08T00:16:17,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03a1dff10ddf4164867bc702e837488d, entries=150, sequenceid=178, filesize=11.9 K 2024-12-08T00:16:17,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c03237de284ce3f116a5db8c2dd8a9d8 in 561ms, sequenceid=178, compaction requested=true 2024-12-08T00:16:17,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:17,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:17,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:17,786 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:17,786 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:17,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:17,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:17,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:17,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:17,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:17,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:17,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/B is initiating minor compaction (all files) 2024-12-08T00:16:17,787 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/A is initiating minor compaction (all files) 2024-12-08T00:16:17,787 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/A in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,787 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/B in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,787 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/cf2e8464bc2041f594c999f47e162561, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/241a749e0f9847198350d52ccd3cccda, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/3808e2bf214941e5a70116a2a311e9a9] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=35.8 K 2024-12-08T00:16:17,787 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ca5726e99be8401f8f08af2365525009, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/11b53de74c5645a2a98e40e3ca1b2f71, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/eaea035b51c04dd2ba3290b39c23b11d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=35.8 K 2024-12-08T00:16:17,788 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca5726e99be8401f8f08af2365525009, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616976131 2024-12-08T00:16:17,788 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2e8464bc2041f594c999f47e162561, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616976131 2024-12-08T00:16:17,788 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 241a749e0f9847198350d52ccd3cccda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733616976203 2024-12-08T00:16:17,788 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b53de74c5645a2a98e40e3ca1b2f71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733616976203 2024-12-08T00:16:17,788 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 3808e2bf214941e5a70116a2a311e9a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733616976881 2024-12-08T00:16:17,788 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaea035b51c04dd2ba3290b39c23b11d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733616976881 2024-12-08T00:16:17,798 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#A#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:17,799 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/226c2aacfe774dc8af9cf8ebc97a6d2d is 50, key is test_row_0/A:col10/1733616976881/Put/seqid=0 2024-12-08T00:16:17,799 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#B#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:17,800 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/dd04d02effdb48d88d3ae13346d37798 is 50, key is test_row_0/B:col10/1733616976881/Put/seqid=0 2024-12-08T00:16:17,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742269_1445 (size=12561) 2024-12-08T00:16:17,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742270_1446 (size=12561) 2024-12-08T00:16:17,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T00:16:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:17,925 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T00:16:17,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:17,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:17,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:17,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:17,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:17,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:17,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/1ead373bf038431cb6ebcdf6e4812c15 is 50, key is test_row_0/A:col10/1733616977273/Put/seqid=0 2024-12-08T00:16:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T00:16:17,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:17,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742271_1447 (size=12151) 2024-12-08T00:16:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617037956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617037958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617037964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:17,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617037966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:17,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617038081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617038081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617038081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617038081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,214 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/226c2aacfe774dc8af9cf8ebc97a6d2d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/226c2aacfe774dc8af9cf8ebc97a6d2d 2024-12-08T00:16:18,214 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/dd04d02effdb48d88d3ae13346d37798 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/dd04d02effdb48d88d3ae13346d37798 2024-12-08T00:16:18,218 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/A of c03237de284ce3f116a5db8c2dd8a9d8 into 226c2aacfe774dc8af9cf8ebc97a6d2d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:18,218 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/B of c03237de284ce3f116a5db8c2dd8a9d8 into dd04d02effdb48d88d3ae13346d37798(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:18,218 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:18,218 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:18,218 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/A, priority=13, startTime=1733616977786; duration=0sec 2024-12-08T00:16:18,218 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/B, priority=13, startTime=1733616977786; duration=0sec 2024-12-08T00:16:18,218 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:18,218 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:18,218 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:A 2024-12-08T00:16:18,218 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:B 2024-12-08T00:16:18,218 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:18,219 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:18,219 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/C is initiating minor compaction (all files) 2024-12-08T00:16:18,219 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/C in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:18,219 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/63762f72a5ff4f5da7338f41384f5cf5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/f48812e699e04ed5beaacb89984a5724, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03a1dff10ddf4164867bc702e837488d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=35.8 K 2024-12-08T00:16:18,220 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63762f72a5ff4f5da7338f41384f5cf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733616976131 2024-12-08T00:16:18,220 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting f48812e699e04ed5beaacb89984a5724, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733616976203 2024-12-08T00:16:18,220 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03a1dff10ddf4164867bc702e837488d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733616976881 2024-12-08T00:16:18,226 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:18,227 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/a9bbe5b4801840819ab2bf9f9da57f07 is 50, key is test_row_0/C:col10/1733616976881/Put/seqid=0 2024-12-08T00:16:18,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742272_1448 (size=12561) 2024-12-08T00:16:18,242 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/a9bbe5b4801840819ab2bf9f9da57f07 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a9bbe5b4801840819ab2bf9f9da57f07 2024-12-08T00:16:18,246 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/C of c03237de284ce3f116a5db8c2dd8a9d8 into a9bbe5b4801840819ab2bf9f9da57f07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:18,246 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:18,246 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/C, priority=13, startTime=1733616977786; duration=0sec 2024-12-08T00:16:18,247 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:18,247 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:C 2024-12-08T00:16:18,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617038291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617038291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617038291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617038301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,341 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/1ead373bf038431cb6ebcdf6e4812c15 2024-12-08T00:16:18,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/0d16fc0e68c1448eac844ea6753c4617 is 50, key is test_row_0/B:col10/1733616977273/Put/seqid=0 2024-12-08T00:16:18,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742273_1449 (size=12151) 2024-12-08T00:16:18,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617038421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,424 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in c03237de284ce3f116a5db8c2dd8a9d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T00:16:18,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617038601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617038601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617038601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617038610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:18,750 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/0d16fc0e68c1448eac844ea6753c4617 2024-12-08T00:16:18,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/e3771857aaec47cb9f7ef3101820da2e is 50, key is test_row_0/C:col10/1733616977273/Put/seqid=0 2024-12-08T00:16:18,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742274_1450 (size=12151) 2024-12-08T00:16:19,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617039111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617039111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617039111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617039131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:19,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/e3771857aaec47cb9f7ef3101820da2e 2024-12-08T00:16:19,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/1ead373bf038431cb6ebcdf6e4812c15 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/1ead373bf038431cb6ebcdf6e4812c15 2024-12-08T00:16:19,165 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/1ead373bf038431cb6ebcdf6e4812c15, entries=150, sequenceid=198, filesize=11.9 K 2024-12-08T00:16:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/0d16fc0e68c1448eac844ea6753c4617 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0d16fc0e68c1448eac844ea6753c4617 2024-12-08T00:16:19,168 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0d16fc0e68c1448eac844ea6753c4617, entries=150, sequenceid=198, filesize=11.9 K 2024-12-08T00:16:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/e3771857aaec47cb9f7ef3101820da2e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e3771857aaec47cb9f7ef3101820da2e 2024-12-08T00:16:19,171 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e3771857aaec47cb9f7ef3101820da2e, entries=150, sequenceid=198, filesize=11.9 K 2024-12-08T00:16:19,172 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c03237de284ce3f116a5db8c2dd8a9d8 in 1248ms, sequenceid=198, compaction requested=false 2024-12-08T00:16:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-08T00:16:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-08T00:16:19,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-08T00:16:19,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-12-08T00:16:19,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8730 sec 2024-12-08T00:16:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T00:16:19,451 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:19,451 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-08T00:16:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T00:16:19,453 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:19,453 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:19,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T00:16:19,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T00:16:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:19,606 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:16:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/5583cdbc24f6473c8059853a95818afd is 50, key is test_row_0/A:col10/1733616977944/Put/seqid=0 2024-12-08T00:16:19,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742275_1451 (size=12151) 2024-12-08T00:16:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T00:16:20,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/5583cdbc24f6473c8059853a95818afd 2024-12-08T00:16:20,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/eea9efac62614682b5c34fb5fc81964e is 50, key is test_row_0/B:col10/1733616977944/Put/seqid=0 2024-12-08T00:16:20,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742276_1452 (size=12151) 2024-12-08T00:16:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T00:16:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:20,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:20,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617040162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617040162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617040166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617040171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617040281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617040282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617040282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617040282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,424 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/eea9efac62614682b5c34fb5fc81964e 2024-12-08T00:16:20,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/c6d292d789cd4c9ba430827710d281a0 is 50, key is test_row_0/C:col10/1733616977944/Put/seqid=0 2024-12-08T00:16:20,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742277_1453 (size=12151) 2024-12-08T00:16:20,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617040451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,456 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in c03237de284ce3f116a5db8c2dd8a9d8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617040490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617040490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617040501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617040501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T00:16:20,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617040811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617040811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617040811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617040812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:20,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/c6d292d789cd4c9ba430827710d281a0 2024-12-08T00:16:20,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/5583cdbc24f6473c8059853a95818afd as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/5583cdbc24f6473c8059853a95818afd 2024-12-08T00:16:20,842 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/5583cdbc24f6473c8059853a95818afd, entries=150, sequenceid=214, filesize=11.9 K 2024-12-08T00:16:20,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/eea9efac62614682b5c34fb5fc81964e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/eea9efac62614682b5c34fb5fc81964e 2024-12-08T00:16:20,846 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/eea9efac62614682b5c34fb5fc81964e, entries=150, sequenceid=214, filesize=11.9 K 2024-12-08T00:16:20,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/c6d292d789cd4c9ba430827710d281a0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/c6d292d789cd4c9ba430827710d281a0 2024-12-08T00:16:20,849 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/c6d292d789cd4c9ba430827710d281a0, entries=150, sequenceid=214, filesize=11.9 K 2024-12-08T00:16:20,849 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c03237de284ce3f116a5db8c2dd8a9d8 in 1244ms, sequenceid=214, compaction requested=true 2024-12-08T00:16:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-08T00:16:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-08T00:16:20,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-08T00:16:20,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3970 sec 2024-12-08T00:16:20,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4010 sec 2024-12-08T00:16:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:21,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:16:21,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:21,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:21,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:21,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:21,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:21,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:21,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/82b8f36b6b8849b689fbaa110bb034b5 is 50, key is test_row_0/A:col10/1733616980169/Put/seqid=0 2024-12-08T00:16:21,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742278_1454 (size=14541) 2024-12-08T00:16:21,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617041348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617041348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617041349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617041350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617041460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617041461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617041461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617041461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T00:16:21,611 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:21,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-08T00:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T00:16:21,614 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:21,614 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:21,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:21,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617041671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617041671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617041671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617041671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T00:16:21,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/82b8f36b6b8849b689fbaa110bb034b5 2024-12-08T00:16:21,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/370851bcf2a24c0db3603bbf48f7541a is 50, key is test_row_0/B:col10/1733616980169/Put/seqid=0 2024-12-08T00:16:21,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742279_1455 (size=12151) 2024-12-08T00:16:21,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/370851bcf2a24c0db3603bbf48f7541a 2024-12-08T00:16:21,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T00:16:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:21,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:21,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/decd78f34fc1422fb3711bd9aa65820d is 50, key is test_row_0/C:col10/1733616980169/Put/seqid=0 2024-12-08T00:16:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:21,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742280_1456 (size=12151) 2024-12-08T00:16:21,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/decd78f34fc1422fb3711bd9aa65820d 2024-12-08T00:16:21,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/82b8f36b6b8849b689fbaa110bb034b5 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/82b8f36b6b8849b689fbaa110bb034b5 2024-12-08T00:16:21,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/82b8f36b6b8849b689fbaa110bb034b5, entries=200, sequenceid=234, filesize=14.2 K 2024-12-08T00:16:21,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/370851bcf2a24c0db3603bbf48f7541a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/370851bcf2a24c0db3603bbf48f7541a 2024-12-08T00:16:21,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/370851bcf2a24c0db3603bbf48f7541a, entries=150, sequenceid=234, filesize=11.9 K 2024-12-08T00:16:21,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/decd78f34fc1422fb3711bd9aa65820d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/decd78f34fc1422fb3711bd9aa65820d 2024-12-08T00:16:21,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/decd78f34fc1422fb3711bd9aa65820d, entries=150, sequenceid=234, filesize=11.9 K 2024-12-08T00:16:21,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c03237de284ce3f116a5db8c2dd8a9d8 in 478ms, sequenceid=234, compaction requested=true 2024-12-08T00:16:21,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:21,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:21,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:21,806 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:21,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:21,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:21,806 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:21,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:21,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:21,806 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:21,806 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/A is initiating minor compaction (all files) 2024-12-08T00:16:21,806 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/A in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:21,807 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/226c2aacfe774dc8af9cf8ebc97a6d2d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/1ead373bf038431cb6ebcdf6e4812c15, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/5583cdbc24f6473c8059853a95818afd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/82b8f36b6b8849b689fbaa110bb034b5] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=50.2 K 2024-12-08T00:16:21,807 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 226c2aacfe774dc8af9cf8ebc97a6d2d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733616976881 2024-12-08T00:16:21,807 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ead373bf038431cb6ebcdf6e4812c15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733616977273 2024-12-08T00:16:21,807 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5583cdbc24f6473c8059853a95818afd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733616977944 2024-12-08T00:16:21,807 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82b8f36b6b8849b689fbaa110bb034b5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733616980160 2024-12-08T00:16:21,809 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:21,809 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/B is initiating minor compaction (all files) 2024-12-08T00:16:21,809 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/B in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:21,809 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/dd04d02effdb48d88d3ae13346d37798, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0d16fc0e68c1448eac844ea6753c4617, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/eea9efac62614682b5c34fb5fc81964e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/370851bcf2a24c0db3603bbf48f7541a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=47.9 K 2024-12-08T00:16:21,810 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting dd04d02effdb48d88d3ae13346d37798, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733616976881 2024-12-08T00:16:21,810 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d16fc0e68c1448eac844ea6753c4617, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733616977273 2024-12-08T00:16:21,810 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting eea9efac62614682b5c34fb5fc81964e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733616977944 2024-12-08T00:16:21,810 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 370851bcf2a24c0db3603bbf48f7541a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733616980160 2024-12-08T00:16:21,816 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:21,817 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/ad1141f43aec4ef2959054ab2d818390 is 50, key is test_row_0/A:col10/1733616980169/Put/seqid=0 2024-12-08T00:16:21,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742281_1457 (size=12697) 2024-12-08T00:16:21,829 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#B#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:21,830 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/53cd3e120484451aa838f34d59f4eaaf is 50, key is test_row_0/B:col10/1733616980169/Put/seqid=0 2024-12-08T00:16:21,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742282_1458 (size=12697) 2024-12-08T00:16:21,843 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/53cd3e120484451aa838f34d59f4eaaf as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/53cd3e120484451aa838f34d59f4eaaf 2024-12-08T00:16:21,849 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/B of c03237de284ce3f116a5db8c2dd8a9d8 into 53cd3e120484451aa838f34d59f4eaaf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:21,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:21,849 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/B, priority=12, startTime=1733616981806; duration=0sec 2024-12-08T00:16:21,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:21,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:B 2024-12-08T00:16:21,849 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:21,852 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:21,852 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/C is initiating minor compaction (all files) 2024-12-08T00:16:21,852 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/C in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:21,853 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a9bbe5b4801840819ab2bf9f9da57f07, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e3771857aaec47cb9f7ef3101820da2e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/c6d292d789cd4c9ba430827710d281a0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/decd78f34fc1422fb3711bd9aa65820d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=47.9 K 2024-12-08T00:16:21,853 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a9bbe5b4801840819ab2bf9f9da57f07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733616976881 2024-12-08T00:16:21,853 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e3771857aaec47cb9f7ef3101820da2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733616977273 2024-12-08T00:16:21,856 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c6d292d789cd4c9ba430827710d281a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733616977944 2024-12-08T00:16:21,856 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting decd78f34fc1422fb3711bd9aa65820d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733616980160 2024-12-08T00:16:21,875 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#C#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:21,875 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/a7e15d7f62c74c8a9e3eafccf30d1ce1 is 50, key is test_row_0/C:col10/1733616980169/Put/seqid=0 2024-12-08T00:16:21,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742283_1459 (size=12697) 2024-12-08T00:16:21,892 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/a7e15d7f62c74c8a9e3eafccf30d1ce1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a7e15d7f62c74c8a9e3eafccf30d1ce1 2024-12-08T00:16:21,897 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/C of c03237de284ce3f116a5db8c2dd8a9d8 into a7e15d7f62c74c8a9e3eafccf30d1ce1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:21,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:21,897 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/C, priority=12, startTime=1733616981806; duration=0sec 2024-12-08T00:16:21,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:21,897 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:C 2024-12-08T00:16:21,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T00:16:21,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:21,921 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T00:16:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T00:16:21,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/06e3288735e5403ebe5e0b839b8bb2bd is 50, key is test_row_0/A:col10/1733616981349/Put/seqid=0 2024-12-08T00:16:21,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742284_1460 (size=12151) 2024-12-08T00:16:21,971 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/06e3288735e5403ebe5e0b839b8bb2bd 2024-12-08T00:16:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:21,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:21,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/b4574edc733e4275bb1e363b2b21035f is 50, key is test_row_0/B:col10/1733616981349/Put/seqid=0 2024-12-08T00:16:22,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742285_1461 (size=12151) 2024-12-08T00:16:22,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617042048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617042050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617042052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617042060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617042171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617042171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617042171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617042180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,227 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/ad1141f43aec4ef2959054ab2d818390 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ad1141f43aec4ef2959054ab2d818390 2024-12-08T00:16:22,234 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/A of c03237de284ce3f116a5db8c2dd8a9d8 into ad1141f43aec4ef2959054ab2d818390(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:22,234 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:22,234 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/A, priority=12, startTime=1733616981805; duration=0sec 2024-12-08T00:16:22,234 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:22,234 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:A 2024-12-08T00:16:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T00:16:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617042381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617042381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617042391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617042391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,410 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/b4574edc733e4275bb1e363b2b21035f 2024-12-08T00:16:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/bbb74647bd3b48538d33c99a290395b0 is 50, key is test_row_0/C:col10/1733616981349/Put/seqid=0 2024-12-08T00:16:22,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742286_1462 (size=12151) 2024-12-08T00:16:22,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617042691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617042691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617042711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617042711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T00:16:22,858 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/bbb74647bd3b48538d33c99a290395b0 2024-12-08T00:16:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/06e3288735e5403ebe5e0b839b8bb2bd as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/06e3288735e5403ebe5e0b839b8bb2bd 2024-12-08T00:16:22,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/06e3288735e5403ebe5e0b839b8bb2bd, entries=150, sequenceid=248, filesize=11.9 K 2024-12-08T00:16:22,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/b4574edc733e4275bb1e363b2b21035f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b4574edc733e4275bb1e363b2b21035f 2024-12-08T00:16:22,881 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b4574edc733e4275bb1e363b2b21035f, entries=150, sequenceid=248, filesize=11.9 K 2024-12-08T00:16:22,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/bbb74647bd3b48538d33c99a290395b0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/bbb74647bd3b48538d33c99a290395b0 2024-12-08T00:16:22,889 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/bbb74647bd3b48538d33c99a290395b0, entries=150, sequenceid=248, filesize=11.9 K 2024-12-08T00:16:22,891 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c03237de284ce3f116a5db8c2dd8a9d8 in 969ms, sequenceid=248, compaction requested=false 2024-12-08T00:16:22,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:22,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:22,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-08T00:16:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-08T00:16:22,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-08T00:16:22,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2800 sec 2024-12-08T00:16:22,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.2870 sec 2024-12-08T00:16:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:23,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T00:16:23,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616982011, max timestamp=1733616983202Num uniques 859; 2024-12-08T00:16:23,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:23,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616982011, max timestamp=1733616983202Num uniques 859; 2024-12-08T00:16:23,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:23,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616982011, max timestamp=1733616983202Num uniques 859; 2024-12-08T00:16:23,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:23,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:23,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/33e49f4be3b84695a50d71c23e9b16f7 is 50, key is test_row_0/A:col10/1733616983204/Put/seqid=0 2024-12-08T00:16:23,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742287_1463 (size=12301) 2024-12-08T00:16:23,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733616983303, max timestamp=1733616983312Num uniques 859; 2024-12-08T00:16:23,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733616983303, max timestamp=1733616983312Num uniques 859; 2024-12-08T00:16:23,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733616983303, max timestamp=1733616983312Num uniques 859; 2024-12-08T00:16:23,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-08T00:16:23,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-08T00:16:23,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-08T00:16:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617043371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617043373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617043385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617043392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617043501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617043502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617043511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617043511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/33e49f4be3b84695a50d71c23e9b16f7 2024-12-08T00:16:23,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/2d1a52522da64a79a88feefb492152ce is 50, key is test_row_0/B:col10/1733616983204/Put/seqid=0 2024-12-08T00:16:23,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742288_1464 (size=12301) 2024-12-08T00:16:23,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617043721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617043721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617043721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617043721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T00:16:23,762 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:23,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-08T00:16:23,764 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T00:16:23,764 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:23,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T00:16:23,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T00:16:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:23,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617044031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617044031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617044041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617044041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T00:16:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/2d1a52522da64a79a88feefb492152ce 2024-12-08T00:16:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T00:16:24,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/fe8e3a28117746f5ab7f6642f7ba1e39 is 50, key is test_row_0/C:col10/1733616983204/Put/seqid=0 2024-12-08T00:16:24,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742289_1465 (size=12301) 2024-12-08T00:16:24,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T00:16:24,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:24,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T00:16:24,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:24,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T00:16:24,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1733617044471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,474 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in c03237de284ce3f116a5db8c2dd8a9d8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/fe8e3a28117746f5ab7f6642f7ba1e39 2024-12-08T00:16:24,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/33e49f4be3b84695a50d71c23e9b16f7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/33e49f4be3b84695a50d71c23e9b16f7 2024-12-08T00:16:24,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/33e49f4be3b84695a50d71c23e9b16f7, entries=150, sequenceid=271, filesize=12.0 K 2024-12-08T00:16:24,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/2d1a52522da64a79a88feefb492152ce as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2d1a52522da64a79a88feefb492152ce 2024-12-08T00:16:24,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2d1a52522da64a79a88feefb492152ce, entries=150, sequenceid=271, filesize=12.0 K 2024-12-08T00:16:24,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/fe8e3a28117746f5ab7f6642f7ba1e39 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/fe8e3a28117746f5ab7f6642f7ba1e39 2024-12-08T00:16:24,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/fe8e3a28117746f5ab7f6642f7ba1e39, entries=150, sequenceid=271, filesize=12.0 K 2024-12-08T00:16:24,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for c03237de284ce3f116a5db8c2dd8a9d8 in 1305ms, sequenceid=271, compaction requested=true 2024-12-08T00:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:24,510 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:24,510 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:24,511 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:24,511 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/A is initiating minor compaction (all files) 2024-12-08T00:16:24,511 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:24,511 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/A in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,511 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/B is initiating minor compaction (all files) 2024-12-08T00:16:24,511 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/B in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,511 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ad1141f43aec4ef2959054ab2d818390, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/06e3288735e5403ebe5e0b839b8bb2bd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/33e49f4be3b84695a50d71c23e9b16f7] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=36.3 K 2024-12-08T00:16:24,511 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/53cd3e120484451aa838f34d59f4eaaf, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b4574edc733e4275bb1e363b2b21035f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2d1a52522da64a79a88feefb492152ce] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=36.3 K 2024-12-08T00:16:24,511 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 53cd3e120484451aa838f34d59f4eaaf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733616980160 2024-12-08T00:16:24,511 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad1141f43aec4ef2959054ab2d818390, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733616980160 2024-12-08T00:16:24,512 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting b4574edc733e4275bb1e363b2b21035f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733616981345 2024-12-08T00:16:24,512 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e3288735e5403ebe5e0b839b8bb2bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733616981345 2024-12-08T00:16:24,512 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d1a52522da64a79a88feefb492152ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733616982011 2024-12-08T00:16:24,512 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33e49f4be3b84695a50d71c23e9b16f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733616982011 2024-12-08T00:16:24,516 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#A#compaction#390 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:24,517 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/d9390969f29143c08adfd79dd149a7a3 is 50, key is test_row_0/A:col10/1733616983204/Put/seqid=0 2024-12-08T00:16:24,517 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#B#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:24,517 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/8ef41cc4e52e423fb32047964c797e79 is 50, key is test_row_0/B:col10/1733616983204/Put/seqid=0 2024-12-08T00:16:24,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742291_1467 (size=12949) 2024-12-08T00:16:24,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742290_1466 (size=12949) 2024-12-08T00:16:24,529 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/d9390969f29143c08adfd79dd149a7a3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/d9390969f29143c08adfd79dd149a7a3 2024-12-08T00:16:24,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T00:16:24,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,533 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-12-08T00:16:24,533 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/A of c03237de284ce3f116a5db8c2dd8a9d8 into d9390969f29143c08adfd79dd149a7a3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:24,533 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:24,533 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/A, priority=13, startTime=1733616984510; duration=0sec 2024-12-08T00:16:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:24,533 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:24,533 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:A 2024-12-08T00:16:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:24,533 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:24,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:24,536 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:24,536 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/C is initiating minor compaction (all files) 2024-12-08T00:16:24,536 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/C in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:24,536 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a7e15d7f62c74c8a9e3eafccf30d1ce1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/bbb74647bd3b48538d33c99a290395b0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/fe8e3a28117746f5ab7f6642f7ba1e39] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=36.3 K 2024-12-08T00:16:24,537 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7e15d7f62c74c8a9e3eafccf30d1ce1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733616980160 2024-12-08T00:16:24,537 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbb74647bd3b48538d33c99a290395b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733616981345 2024-12-08T00:16:24,537 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe8e3a28117746f5ab7f6642f7ba1e39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733616982011 2024-12-08T00:16:24,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/abfa2919fd7a4b39a54f13a9efb106d6 is 50, key is test_row_0/A:col10/1733616983367/Put/seqid=0 2024-12-08T00:16:24,549 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#C#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:24,550 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/7357ba81210848209451eeb2da17e085 is 50, key is test_row_0/C:col10/1733616983204/Put/seqid=0 2024-12-08T00:16:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:24,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:24,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742292_1468 (size=12301) 2024-12-08T00:16:24,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742293_1469 (size=12949) 2024-12-08T00:16:24,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617044561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617044562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617044567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617044568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617044671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617044681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617044681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617044891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617044891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617044891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T00:16:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:24,926 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/8ef41cc4e52e423fb32047964c797e79 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8ef41cc4e52e423fb32047964c797e79 2024-12-08T00:16:24,930 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/B of c03237de284ce3f116a5db8c2dd8a9d8 into 8ef41cc4e52e423fb32047964c797e79(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:24,930 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:24,930 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/B, priority=13, startTime=1733616984510; duration=0sec 2024-12-08T00:16:24,930 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:24,930 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:B 2024-12-08T00:16:24,957 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/abfa2919fd7a4b39a54f13a9efb106d6 2024-12-08T00:16:24,966 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/7357ba81210848209451eeb2da17e085 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7357ba81210848209451eeb2da17e085 2024-12-08T00:16:24,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/88d4f353a6184150b8c2c79ceb519d7b is 50, key is test_row_0/B:col10/1733616983367/Put/seqid=0 2024-12-08T00:16:24,971 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/C of c03237de284ce3f116a5db8c2dd8a9d8 into 7357ba81210848209451eeb2da17e085(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:24,971 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:24,971 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/C, priority=13, startTime=1733616984510; duration=0sec 2024-12-08T00:16:24,971 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:24,971 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:C 2024-12-08T00:16:24,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742294_1470 (size=12301) 2024-12-08T00:16:24,992 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/88d4f353a6184150b8c2c79ceb519d7b 2024-12-08T00:16:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/3b3aec1691754164a757a1ff48d97651 is 50, key is test_row_0/C:col10/1733616983367/Put/seqid=0 2024-12-08T00:16:25,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742295_1471 (size=12301) 2024-12-08T00:16:25,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617045212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617045213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617045213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,403 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/3b3aec1691754164a757a1ff48d97651 2024-12-08T00:16:25,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/abfa2919fd7a4b39a54f13a9efb106d6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/abfa2919fd7a4b39a54f13a9efb106d6 2024-12-08T00:16:25,410 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/abfa2919fd7a4b39a54f13a9efb106d6, entries=150, sequenceid=308, filesize=12.0 K 2024-12-08T00:16:25,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/88d4f353a6184150b8c2c79ceb519d7b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/88d4f353a6184150b8c2c79ceb519d7b 2024-12-08T00:16:25,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/88d4f353a6184150b8c2c79ceb519d7b, entries=150, sequenceid=308, filesize=12.0 K 2024-12-08T00:16:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/3b3aec1691754164a757a1ff48d97651 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/3b3aec1691754164a757a1ff48d97651 2024-12-08T00:16:25,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/3b3aec1691754164a757a1ff48d97651, entries=150, sequenceid=308, filesize=12.0 K 2024-12-08T00:16:25,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for c03237de284ce3f116a5db8c2dd8a9d8 in 900ms, sequenceid=308, compaction requested=false 2024-12-08T00:16:25,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:25,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:25,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-08T00:16:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-08T00:16:25,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-08T00:16:25,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6700 sec 2024-12-08T00:16:25,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.6750 sec 2024-12-08T00:16:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:25,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:16:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:25,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/c2b4f023249941a1aece0a6cb2b28916 is 50, key is test_row_0/A:col10/1733616984566/Put/seqid=0 2024-12-08T00:16:25,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742296_1472 (size=17181) 2024-12-08T00:16:25,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616985638, max timestamp=1733616985668Num uniques 859; 2024-12-08T00:16:25,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616985638, max timestamp=1733616985668Num uniques 859; 2024-12-08T00:16:25,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:25,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:25,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616985638, max timestamp=1733616985668Num uniques 859; 2024-12-08T00:16:25,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:25,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733616985722, max timestamp=1733616985757Num uniques 1018; 2024-12-08T00:16:25,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-08T00:16:25,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733616985722, max timestamp=1733616985757Num uniques 1018; 2024-12-08T00:16:25,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-08T00:16:25,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733616985722, max timestamp=1733616985757Num uniques 1018; 2024-12-08T00:16:25,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-08T00:16:25,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617045787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617045790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617045791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617045793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617045900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617045900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T00:16:25,911 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:25,911 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-08T00:16:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T00:16:25,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617045911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,914 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617045911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:25,916 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:25,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:25,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/c2b4f023249941a1aece0a6cb2b28916 2024-12-08T00:16:25,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/5d29cd4cfe62477aa55e0c9a1f05340a is 50, key is test_row_0/B:col10/1733616984566/Put/seqid=0 2024-12-08T00:16:26,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742297_1473 (size=12301) 2024-12-08T00:16:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T00:16:26,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T00:16:26,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:26,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617046120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617046120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:26,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617046120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:26,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617046120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:26,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T00:16:26,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:26,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T00:16:26,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T00:16:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/5d29cd4cfe62477aa55e0c9a1f05340a 2024-12-08T00:16:26,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/53f80c9a1269459eb1d7e60200c58c94 is 50, key is test_row_0/C:col10/1733616984566/Put/seqid=0 2024-12-08T00:16:26,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742298_1474 (size=12301) 2024-12-08T00:16:26,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617046441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617046441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:26,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617046442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:26,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:26,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617046442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:26,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:26,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:26,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T00:16:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T00:16:26,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T00:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:26,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/53f80c9a1269459eb1d7e60200c58c94 2024-12-08T00:16:26,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/c2b4f023249941a1aece0a6cb2b28916 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c2b4f023249941a1aece0a6cb2b28916 2024-12-08T00:16:26,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c2b4f023249941a1aece0a6cb2b28916, entries=250, sequenceid=320, filesize=16.8 K 2024-12-08T00:16:26,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/5d29cd4cfe62477aa55e0c9a1f05340a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/5d29cd4cfe62477aa55e0c9a1f05340a 2024-12-08T00:16:26,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/5d29cd4cfe62477aa55e0c9a1f05340a, entries=150, sequenceid=320, filesize=12.0 K 2024-12-08T00:16:26,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/53f80c9a1269459eb1d7e60200c58c94 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/53f80c9a1269459eb1d7e60200c58c94 2024-12-08T00:16:26,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/53f80c9a1269459eb1d7e60200c58c94, entries=150, sequenceid=320, filesize=12.0 K 2024-12-08T00:16:26,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c03237de284ce3f116a5db8c2dd8a9d8 in 1258ms, sequenceid=320, compaction requested=true 2024-12-08T00:16:26,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:26,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:26,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:26,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:26,843 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:26,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:26,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:26,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:26,843 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:26,844 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:26,844 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/B is initiating minor compaction (all files) 2024-12-08T00:16:26,844 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/B in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,844 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8ef41cc4e52e423fb32047964c797e79, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/88d4f353a6184150b8c2c79ceb519d7b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/5d29cd4cfe62477aa55e0c9a1f05340a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=36.7 K 2024-12-08T00:16:26,844 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:26,844 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/A is initiating minor compaction (all files) 2024-12-08T00:16:26,845 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/A in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,845 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/d9390969f29143c08adfd79dd149a7a3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/abfa2919fd7a4b39a54f13a9efb106d6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c2b4f023249941a1aece0a6cb2b28916] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=41.4 K 2024-12-08T00:16:26,845 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d9390969f29143c08adfd79dd149a7a3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733616982011 2024-12-08T00:16:26,845 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef41cc4e52e423fb32047964c797e79, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733616982011 2024-12-08T00:16:26,845 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting abfa2919fd7a4b39a54f13a9efb106d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733616983348 2024-12-08T00:16:26,845 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d4f353a6184150b8c2c79ceb519d7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733616983348 2024-12-08T00:16:26,846 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b4f023249941a1aece0a6cb2b28916, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733616984558 2024-12-08T00:16:26,846 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d29cd4cfe62477aa55e0c9a1f05340a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733616984566 2024-12-08T00:16:26,854 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:26,854 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/0b385178472c4efdb7e54be87ad56ec7 is 50, key is test_row_0/B:col10/1733616984566/Put/seqid=0 2024-12-08T00:16:26,863 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#A#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:26,863 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/81734082bc124f99bc27d4069335ca52 is 50, key is test_row_0/A:col10/1733616984566/Put/seqid=0 2024-12-08T00:16:26,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T00:16:26,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,868 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=134.18 KB heapSize=374.77 KB 2024-12-08T00:16:26,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:26,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/a319262f7e87454e84d0c5e4ed962583 is 50, key is test_row_0/A:col10/1733616985790/Put/seqid=0 2024-12-08T00:16:26,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742299_1475 (size=13051) 2024-12-08T00:16:26,914 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/0b385178472c4efdb7e54be87ad56ec7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0b385178472c4efdb7e54be87ad56ec7 2024-12-08T00:16:26,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742300_1476 (size=13051) 2024-12-08T00:16:26,925 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/B of c03237de284ce3f116a5db8c2dd8a9d8 into 0b385178472c4efdb7e54be87ad56ec7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:26,925 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:26,925 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/B, priority=13, startTime=1733616986843; duration=0sec 2024-12-08T00:16:26,925 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:26,925 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:B 2024-12-08T00:16:26,925 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:26,926 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:26,926 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/C is initiating minor compaction (all files) 2024-12-08T00:16:26,926 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/C in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:26,926 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7357ba81210848209451eeb2da17e085, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/3b3aec1691754164a757a1ff48d97651, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/53f80c9a1269459eb1d7e60200c58c94] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=36.7 K 2024-12-08T00:16:26,928 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7357ba81210848209451eeb2da17e085, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733616982011 2024-12-08T00:16:26,929 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b3aec1691754164a757a1ff48d97651, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733616983348 2024-12-08T00:16:26,931 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53f80c9a1269459eb1d7e60200c58c94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733616984566 2024-12-08T00:16:26,952 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#C#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:26,953 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/4ade9b0e3ac84344aa608fedecddd668 is 50, key is test_row_0/C:col10/1733616984566/Put/seqid=0 2024-12-08T00:16:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:26,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:26,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742301_1477 (size=12301) 2024-12-08T00:16:26,998 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/a319262f7e87454e84d0c5e4ed962583 2024-12-08T00:16:27,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742302_1478 (size=13051) 2024-12-08T00:16:27,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617046990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/966487af894c435cbeaae7e6d39df2f9 is 50, key is test_row_0/B:col10/1733616985790/Put/seqid=0 2024-12-08T00:16:27,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617046999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617047001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617047001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742303_1479 (size=12301) 2024-12-08T00:16:27,039 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/966487af894c435cbeaae7e6d39df2f9 2024-12-08T00:16:27,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/7210e7bec8a348afade70b7f300db675 is 50, key is test_row_0/C:col10/1733616985790/Put/seqid=0 2024-12-08T00:16:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T00:16:27,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742304_1480 (size=12301) 2024-12-08T00:16:27,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/7210e7bec8a348afade70b7f300db675 2024-12-08T00:16:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/a319262f7e87454e84d0c5e4ed962583 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a319262f7e87454e84d0c5e4ed962583 2024-12-08T00:16:27,113 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a319262f7e87454e84d0c5e4ed962583, entries=150, sequenceid=372, filesize=12.0 K 2024-12-08T00:16:27,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/966487af894c435cbeaae7e6d39df2f9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/966487af894c435cbeaae7e6d39df2f9 2024-12-08T00:16:27,117 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/966487af894c435cbeaae7e6d39df2f9, entries=150, sequenceid=372, filesize=12.0 K 2024-12-08T00:16:27,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/7210e7bec8a348afade70b7f300db675 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7210e7bec8a348afade70b7f300db675 2024-12-08T00:16:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617047110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,122 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7210e7bec8a348afade70b7f300db675, entries=150, sequenceid=372, filesize=12.0 K 2024-12-08T00:16:27,123 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~374.72 KB/383712, currentSize=53.67 KB/54960 for c03237de284ce3f116a5db8c2dd8a9d8 in 254ms, sequenceid=372, compaction requested=false 2024-12-08T00:16:27,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:27,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:27,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-08T00:16:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-08T00:16:27,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-08T00:16:27,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2080 sec 2024-12-08T00:16:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:27,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T00:16:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:27,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.2170 sec 2024-12-08T00:16:27,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/f9b0df9eba6544439bb8ef10c72c08fb is 50, key is test_row_0/A:col10/1733616986992/Put/seqid=0 2024-12-08T00:16:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742305_1481 (size=14741) 2024-12-08T00:16:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617047180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617047188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617047190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617047301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617047301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617047311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617047321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,328 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/81734082bc124f99bc27d4069335ca52 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81734082bc124f99bc27d4069335ca52 2024-12-08T00:16:27,332 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/A of c03237de284ce3f116a5db8c2dd8a9d8 into 81734082bc124f99bc27d4069335ca52(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:27,332 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:27,332 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/A, priority=13, startTime=1733616986843; duration=0sec 2024-12-08T00:16:27,332 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:27,332 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:A 2024-12-08T00:16:27,404 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/4ade9b0e3ac84344aa608fedecddd668 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/4ade9b0e3ac84344aa608fedecddd668 2024-12-08T00:16:27,407 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/C of c03237de284ce3f116a5db8c2dd8a9d8 into 4ade9b0e3ac84344aa608fedecddd668(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:27,408 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:27,408 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/C, priority=13, startTime=1733616986843; duration=0sec 2024-12-08T00:16:27,408 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:27,408 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:C 2024-12-08T00:16:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617047510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617047511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617047521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/f9b0df9eba6544439bb8ef10c72c08fb 2024-12-08T00:16:27,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/17988a5b77144911b415440859dadffa is 50, key is test_row_0/B:col10/1733616986992/Put/seqid=0 2024-12-08T00:16:27,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742306_1482 (size=12301) 2024-12-08T00:16:27,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617047631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617047821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617047821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617047830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:27,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/17988a5b77144911b415440859dadffa 2024-12-08T00:16:27,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/59252f0a19244257b16f8a7b7c402000 is 50, key is test_row_0/C:col10/1733616986992/Put/seqid=0 2024-12-08T00:16:27,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742307_1483 (size=12301) 2024-12-08T00:16:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T00:16:28,071 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:28,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-08T00:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T00:16:28,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:28,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:28,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:28,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617048141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T00:16:28,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T00:16:28,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:28,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:28,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:28,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:28,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:28,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617048331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617048341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617048341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:28,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:28,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:28,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/59252f0a19244257b16f8a7b7c402000 2024-12-08T00:16:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/f9b0df9eba6544439bb8ef10c72c08fb as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/f9b0df9eba6544439bb8ef10c72c08fb 2024-12-08T00:16:28,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/f9b0df9eba6544439bb8ef10c72c08fb, entries=200, sequenceid=385, filesize=14.4 K 2024-12-08T00:16:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/17988a5b77144911b415440859dadffa as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/17988a5b77144911b415440859dadffa 2024-12-08T00:16:28,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/17988a5b77144911b415440859dadffa, entries=150, sequenceid=385, filesize=12.0 K 2024-12-08T00:16:28,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/59252f0a19244257b16f8a7b7c402000 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/59252f0a19244257b16f8a7b7c402000 2024-12-08T00:16:28,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/59252f0a19244257b16f8a7b7c402000, entries=150, sequenceid=385, filesize=12.0 K 2024-12-08T00:16:28,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c03237de284ce3f116a5db8c2dd8a9d8 in 1242ms, sequenceid=385, compaction requested=true 2024-12-08T00:16:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:28,372 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:28,372 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:28,373 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:28,373 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/A is initiating minor compaction (all files) 2024-12-08T00:16:28,373 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/A in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:28,373 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81734082bc124f99bc27d4069335ca52, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a319262f7e87454e84d0c5e4ed962583, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/f9b0df9eba6544439bb8ef10c72c08fb] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=39.2 K 2024-12-08T00:16:28,373 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:28,373 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81734082bc124f99bc27d4069335ca52, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733616984566 2024-12-08T00:16:28,373 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/B is initiating minor compaction (all files) 2024-12-08T00:16:28,373 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/B in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:28,373 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0b385178472c4efdb7e54be87ad56ec7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/966487af894c435cbeaae7e6d39df2f9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/17988a5b77144911b415440859dadffa] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=36.8 K 2024-12-08T00:16:28,376 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b385178472c4efdb7e54be87ad56ec7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733616984566 2024-12-08T00:16:28,376 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting a319262f7e87454e84d0c5e4ed962583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733616985789 2024-12-08T00:16:28,376 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 966487af894c435cbeaae7e6d39df2f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733616985789 2024-12-08T00:16:28,378 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9b0df9eba6544439bb8ef10c72c08fb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733616986959 2024-12-08T00:16:28,378 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 17988a5b77144911b415440859dadffa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733616986959 2024-12-08T00:16:28,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T00:16:28,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:28,380 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:16:28,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:28,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:28,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:28,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:28,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:28,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T00:16:28,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/084e5b9e6d2146bd9a831523481a8ed1 is 50, key is test_row_0/A:col10/1733616987188/Put/seqid=0 2024-12-08T00:16:28,419 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:28,419 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/712634942473453f9037a7b27bb228e0 is 50, key is test_row_0/B:col10/1733616986992/Put/seqid=0 2024-12-08T00:16:28,423 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:28,424 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/753bba5fe84b4fa08283b4fad437a45b is 50, key is test_row_0/A:col10/1733616986992/Put/seqid=0 2024-12-08T00:16:28,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742309_1485 (size=13153) 2024-12-08T00:16:28,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742308_1484 (size=12301) 2024-12-08T00:16:28,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/084e5b9e6d2146bd9a831523481a8ed1 2024-12-08T00:16:28,443 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/712634942473453f9037a7b27bb228e0 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/712634942473453f9037a7b27bb228e0 2024-12-08T00:16:28,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742310_1486 (size=13153) 2024-12-08T00:16:28,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/6fb8d0acb4f64ea2a135a8e2ec66d26a is 50, key is test_row_0/B:col10/1733616987188/Put/seqid=0 2024-12-08T00:16:28,458 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/B of c03237de284ce3f116a5db8c2dd8a9d8 into 712634942473453f9037a7b27bb228e0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:28,458 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:28,458 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/B, priority=13, startTime=1733616988372; duration=0sec 2024-12-08T00:16:28,458 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:28,458 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:B 2024-12-08T00:16:28,458 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:28,463 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/753bba5fe84b4fa08283b4fad437a45b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/753bba5fe84b4fa08283b4fad437a45b 2024-12-08T00:16:28,463 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:28,463 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/C is initiating minor compaction (all files) 2024-12-08T00:16:28,463 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/C in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:28,464 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/4ade9b0e3ac84344aa608fedecddd668, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7210e7bec8a348afade70b7f300db675, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/59252f0a19244257b16f8a7b7c402000] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=36.8 K 2024-12-08T00:16:28,464 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ade9b0e3ac84344aa608fedecddd668, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733616984566 2024-12-08T00:16:28,464 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 7210e7bec8a348afade70b7f300db675, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733616985789 2024-12-08T00:16:28,464 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 59252f0a19244257b16f8a7b7c402000, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733616986959 2024-12-08T00:16:28,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742311_1487 (size=12301) 2024-12-08T00:16:28,469 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/A of c03237de284ce3f116a5db8c2dd8a9d8 into 753bba5fe84b4fa08283b4fad437a45b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:28,469 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:28,469 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/A, priority=13, startTime=1733616988372; duration=0sec 2024-12-08T00:16:28,469 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:28,469 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:A 2024-12-08T00:16:28,469 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/6fb8d0acb4f64ea2a135a8e2ec66d26a 2024-12-08T00:16:28,478 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#C#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:28,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/68d1317ad176480480b997d4790f0ace is 50, key is test_row_0/C:col10/1733616987188/Put/seqid=0 2024-12-08T00:16:28,479 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/706e6be87a344e6f911350657c60d680 is 50, key is test_row_0/C:col10/1733616986992/Put/seqid=0 2024-12-08T00:16:28,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742312_1488 (size=13153) 2024-12-08T00:16:28,494 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/706e6be87a344e6f911350657c60d680 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/706e6be87a344e6f911350657c60d680 2024-12-08T00:16:28,501 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/C of c03237de284ce3f116a5db8c2dd8a9d8 into 706e6be87a344e6f911350657c60d680(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:28,501 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:28,501 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/C, priority=13, startTime=1733616988372; duration=0sec 2024-12-08T00:16:28,501 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:28,501 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:C 2024-12-08T00:16:28,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742313_1489 (size=12301) 2024-12-08T00:16:28,513 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/68d1317ad176480480b997d4790f0ace 2024-12-08T00:16:28,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/084e5b9e6d2146bd9a831523481a8ed1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/084e5b9e6d2146bd9a831523481a8ed1 2024-12-08T00:16:28,530 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/084e5b9e6d2146bd9a831523481a8ed1, entries=150, sequenceid=407, filesize=12.0 K 2024-12-08T00:16:28,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/6fb8d0acb4f64ea2a135a8e2ec66d26a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6fb8d0acb4f64ea2a135a8e2ec66d26a 2024-12-08T00:16:28,540 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6fb8d0acb4f64ea2a135a8e2ec66d26a, entries=150, sequenceid=407, filesize=12.0 K 2024-12-08T00:16:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-08T00:16:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/68d1317ad176480480b997d4790f0ace as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/68d1317ad176480480b997d4790f0ace 2024-12-08T00:16:28,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/68d1317ad176480480b997d4790f0ace, entries=150, sequenceid=407, filesize=12.0 K 2024-12-08T00:16:28,546 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for c03237de284ce3f116a5db8c2dd8a9d8 in 166ms, sequenceid=407, compaction requested=false 2024-12-08T00:16:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-08T00:16:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-08T00:16:28,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-08T00:16:28,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 474 msec 2024-12-08T00:16:28,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 485 msec 2024-12-08T00:16:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T00:16:28,701 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:28,701 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-08T00:16:28,704 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T00:16:28,704 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:28,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T00:16:28,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T00:16:28,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:28,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:28,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:28,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-08T00:16:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-08T00:16:28,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-08T00:16:28,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-08T00:16:28,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 160 msec 2024-12-08T00:16:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T00:16:29,021 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:29,021 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-08T00:16:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T00:16:29,024 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:29,025 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:29,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T00:16:29,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T00:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:29,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T00:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/0a5830db66844f85a5da134b19573f46 is 50, key is test_row_1/A:col10/1733616989165/Put/seqid=0 2024-12-08T00:16:29,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. as already flushing 2024-12-08T00:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:29,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742314_1490 (size=12297) 2024-12-08T00:16:29,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616989291, max timestamp=1733616989311Num uniques 859; 2024-12-08T00:16:29,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:29,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616989291, max timestamp=1733616989311Num uniques 859; 2024-12-08T00:16:29,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:29,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733616989291, max timestamp=1733616989311Num uniques 859; 2024-12-08T00:16:29,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T00:16:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T00:16:29,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733616989370, max timestamp=1733616989388Num uniques 1018; 2024-12-08T00:16:29,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-08T00:16:29,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733616989370, max timestamp=1733616989388Num uniques 1018; 2024-12-08T00:16:29,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-08T00:16:29,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733616989370, max timestamp=1733616989388Num uniques 1018; 2024-12-08T00:16:29,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-08T00:16:29,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617049442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617049442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617049449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617049450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617049561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617049561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617049572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617049572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,599 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/0a5830db66844f85a5da134b19573f46 2024-12-08T00:16:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/13e0869c59f54d318069b2f450826b47 is 50, key is test_row_1/B:col10/1733616989165/Put/seqid=0 2024-12-08T00:16:29,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742315_1491 (size=9857) 2024-12-08T00:16:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T00:16:29,714 INFO [Thread-1858 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1858. 2024-12-08T00:16:29,714 DEBUG [Thread-1858 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:29,714 DEBUG [Thread-1858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,714 DEBUG [Thread-1858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:29,717 INFO [Thread-1862 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1862. 2024-12-08T00:16:29,717 DEBUG [Thread-1862 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:29,717 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,717 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:29,719 INFO [Thread-1860 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1860. 2024-12-08T00:16:29,719 DEBUG [Thread-1860 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:29,719 DEBUG [Thread-1860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,719 DEBUG [Thread-1860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:29,723 INFO [Thread-1864 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1864. 2024-12-08T00:16:29,724 DEBUG [Thread-1864 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:29,724 DEBUG [Thread-1864 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,724 DEBUG [Thread-1864 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:29,726 INFO [Thread-1856 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1856. 2024-12-08T00:16:29,726 DEBUG [Thread-1856 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:29,726 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,726 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:29,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617049771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617049771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617049791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617049791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:29,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:29,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:29,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:30,032 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/13e0869c59f54d318069b2f450826b47 2024-12-08T00:16:30,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/d6b1a338ce124ea6b9054a8bcd288499 is 50, key is test_row_1/C:col10/1733616989165/Put/seqid=0 2024-12-08T00:16:30,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742316_1492 (size=9857) 2024-12-08T00:16:30,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:30,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38918 deadline: 1733617050081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38910 deadline: 1733617050081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38822 deadline: 1733617050100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38878 deadline: 1733617050100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 , the old value is region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=2 because the exception is null or not the one we care about 2024-12-08T00:16:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T00:16:30,442 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/d6b1a338ce124ea6b9054a8bcd288499 2024-12-08T00:16:30,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/0a5830db66844f85a5da134b19573f46 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/0a5830db66844f85a5da134b19573f46 2024-12-08T00:16:30,451 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/0a5830db66844f85a5da134b19573f46, entries=150, sequenceid=419, filesize=12.0 K 2024-12-08T00:16:30,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/13e0869c59f54d318069b2f450826b47 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/13e0869c59f54d318069b2f450826b47 2024-12-08T00:16:30,455 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/13e0869c59f54d318069b2f450826b47, entries=100, sequenceid=419, filesize=9.6 K 2024-12-08T00:16:30,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/d6b1a338ce124ea6b9054a8bcd288499 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/d6b1a338ce124ea6b9054a8bcd288499 2024-12-08T00:16:30,460 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/d6b1a338ce124ea6b9054a8bcd288499, entries=100, sequenceid=419, filesize=9.6 K 2024-12-08T00:16:30,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for c03237de284ce3f116a5db8c2dd8a9d8 in 1283ms, sequenceid=419, compaction requested=true 2024-12-08T00:16:30,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:30,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:30,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-08T00:16:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-08T00:16:30,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-08T00:16:30,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4380 sec 2024-12-08T00:16:30,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4440 sec 2024-12-08T00:16:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:30,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=161.02 KB heapSize=455.16 KB 2024-12-08T00:16:30,593 INFO [Thread-1845 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1845. 2024-12-08T00:16:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:30,593 DEBUG [Thread-1845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:30,593 DEBUG [Thread-1845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T00:16:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:30,593 DEBUG [Thread-1845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T00:16:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T00:16:30,594 INFO [Thread-1851 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1851. 2024-12-08T00:16:30,594 DEBUG [Thread-1851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:30,594 DEBUG [Thread-1851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:30,594 DEBUG [Thread-1851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:30,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:30,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:30,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/9df19fc5dce74f608e6ee16754b9964c is 50, key is test_row_0/A:col10/1733616989449/Put/seqid=0 2024-12-08T00:16:30,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742317_1493 (size=12301) 2024-12-08T00:16:30,612 INFO [Thread-1853 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1853. 2024-12-08T00:16:30,612 INFO [Thread-1847 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1847. 2024-12-08T00:16:30,612 DEBUG [Thread-1847 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:30,612 DEBUG [Thread-1853 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:30,612 DEBUG [Thread-1853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:30,612 DEBUG [Thread-1847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:30,613 DEBUG [Thread-1847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:30,613 DEBUG [Thread-1853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:30,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:30,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:31,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/9df19fc5dce74f608e6ee16754b9964c 2024-12-08T00:16:31,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/960c6251ef6f43938980c02559f74156 is 50, key is test_row_0/B:col10/1733616989449/Put/seqid=0 2024-12-08T00:16:31,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742318_1494 (size=12301) 2024-12-08T00:16:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T00:16:31,181 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:31,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/960c6251ef6f43938980c02559f74156 2024-12-08T00:16:31,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/88b2038363e34a1f9bf3ce1cf00b0324 is 50, key is test_row_0/C:col10/1733616989449/Put/seqid=0 2024-12-08T00:16:31,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742319_1495 (size=12301) 2024-12-08T00:16:31,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/88b2038363e34a1f9bf3ce1cf00b0324 2024-12-08T00:16:31,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/9df19fc5dce74f608e6ee16754b9964c as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/9df19fc5dce74f608e6ee16754b9964c 2024-12-08T00:16:31,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/9df19fc5dce74f608e6ee16754b9964c, entries=150, sequenceid=475, filesize=12.0 K 2024-12-08T00:16:31,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/960c6251ef6f43938980c02559f74156 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/960c6251ef6f43938980c02559f74156 2024-12-08T00:16:31,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/960c6251ef6f43938980c02559f74156, entries=150, sequenceid=475, filesize=12.0 K 2024-12-08T00:16:31,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/88b2038363e34a1f9bf3ce1cf00b0324 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88b2038363e34a1f9bf3ce1cf00b0324 2024-12-08T00:16:31,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88b2038363e34a1f9bf3ce1cf00b0324, entries=150, sequenceid=475, filesize=12.0 K 2024-12-08T00:16:31,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~455.11 KB/466032, currentSize=13.42 KB/13740 for c03237de284ce3f116a5db8c2dd8a9d8 in 1264ms, sequenceid=475, compaction requested=true 2024-12-08T00:16:31,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:31,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:31,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:31,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:31,858 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:31,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:31,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c03237de284ce3f116a5db8c2dd8a9d8:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:31,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:31,858 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:31,859 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:31,859 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50052 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:31,859 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/A is initiating minor compaction (all files) 2024-12-08T00:16:31,859 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/B is initiating minor compaction (all files) 2024-12-08T00:16:31,859 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/A in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:31,859 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/B in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:31,859 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/712634942473453f9037a7b27bb228e0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6fb8d0acb4f64ea2a135a8e2ec66d26a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/13e0869c59f54d318069b2f450826b47, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/960c6251ef6f43938980c02559f74156] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=46.5 K 2024-12-08T00:16:31,859 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/753bba5fe84b4fa08283b4fad437a45b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/084e5b9e6d2146bd9a831523481a8ed1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/0a5830db66844f85a5da134b19573f46, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/9df19fc5dce74f608e6ee16754b9964c] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=48.9 K 2024-12-08T00:16:31,859 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 753bba5fe84b4fa08283b4fad437a45b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733616986959 2024-12-08T00:16:31,859 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 712634942473453f9037a7b27bb228e0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733616986959 2024-12-08T00:16:31,860 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 084e5b9e6d2146bd9a831523481a8ed1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733616987160 2024-12-08T00:16:31,860 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fb8d0acb4f64ea2a135a8e2ec66d26a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733616987160 2024-12-08T00:16:31,860 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a5830db66844f85a5da134b19573f46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733616989165 2024-12-08T00:16:31,860 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 13e0869c59f54d318069b2f450826b47, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733616989165 2024-12-08T00:16:31,860 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9df19fc5dce74f608e6ee16754b9964c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733616989449 2024-12-08T00:16:31,860 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 960c6251ef6f43938980c02559f74156, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733616989449 2024-12-08T00:16:31,867 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:31,867 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:31,868 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/dc577eb85bab4d23a63eee41474451e1 is 50, key is test_row_0/A:col10/1733616989449/Put/seqid=0 2024-12-08T00:16:31,868 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/925e0b03a5bd4b08ad1bf7b4aca370ef is 50, key is test_row_0/B:col10/1733616989449/Put/seqid=0 2024-12-08T00:16:31,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742321_1497 (size=13289) 2024-12-08T00:16:31,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742320_1496 (size=13289) 2024-12-08T00:16:32,280 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/925e0b03a5bd4b08ad1bf7b4aca370ef as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/925e0b03a5bd4b08ad1bf7b4aca370ef 2024-12-08T00:16:32,280 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/dc577eb85bab4d23a63eee41474451e1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/dc577eb85bab4d23a63eee41474451e1 2024-12-08T00:16:32,286 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/A of c03237de284ce3f116a5db8c2dd8a9d8 into dc577eb85bab4d23a63eee41474451e1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:32,286 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/B of c03237de284ce3f116a5db8c2dd8a9d8 into 925e0b03a5bd4b08ad1bf7b4aca370ef(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:32,286 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:32,286 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:32,286 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/B, priority=12, startTime=1733616991858; duration=0sec 2024-12-08T00:16:32,286 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/A, priority=12, startTime=1733616991857; duration=0sec 2024-12-08T00:16:32,286 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:32,286 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:32,286 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:A 2024-12-08T00:16:32,287 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:B 2024-12-08T00:16:32,287 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:32,288 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:32,288 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): c03237de284ce3f116a5db8c2dd8a9d8/C is initiating minor compaction (all files) 2024-12-08T00:16:32,289 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c03237de284ce3f116a5db8c2dd8a9d8/C in TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:32,289 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/706e6be87a344e6f911350657c60d680, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/68d1317ad176480480b997d4790f0ace, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/d6b1a338ce124ea6b9054a8bcd288499, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88b2038363e34a1f9bf3ce1cf00b0324] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp, totalSize=46.5 K 2024-12-08T00:16:32,289 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 706e6be87a344e6f911350657c60d680, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733616986959 2024-12-08T00:16:32,289 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68d1317ad176480480b997d4790f0ace, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733616987160 2024-12-08T00:16:32,290 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6b1a338ce124ea6b9054a8bcd288499, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733616989165 2024-12-08T00:16:32,290 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88b2038363e34a1f9bf3ce1cf00b0324, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733616989449 2024-12-08T00:16:32,298 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c03237de284ce3f116a5db8c2dd8a9d8#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:32,298 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/94cc614dd6804b0f88f913a0359192fe is 50, key is test_row_0/C:col10/1733616989449/Put/seqid=0 2024-12-08T00:16:32,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742322_1498 (size=13289) 2024-12-08T00:16:32,715 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/94cc614dd6804b0f88f913a0359192fe as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/94cc614dd6804b0f88f913a0359192fe 2024-12-08T00:16:32,720 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c03237de284ce3f116a5db8c2dd8a9d8/C of c03237de284ce3f116a5db8c2dd8a9d8 into 94cc614dd6804b0f88f913a0359192fe(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:32,720 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c03237de284ce3f116a5db8c2dd8a9d8: 2024-12-08T00:16:32,721 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., storeName=c03237de284ce3f116a5db8c2dd8a9d8/C, priority=12, startTime=1733616991858; duration=0sec 2024-12-08T00:16:32,721 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:32,721 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c03237de284ce3f116a5db8c2dd8a9d8:C 2024-12-08T00:16:34,494 INFO [Thread-1849 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1849. 2024-12-08T00:16:34,494 DEBUG [Thread-1849 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:34,494 DEBUG [Thread-1849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:34,494 DEBUG [Thread-1849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:34,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1573 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4711 rows 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1567 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4696 rows 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1562 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4683 rows 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1564 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4688 rows 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1552 2024-12-08T00:16:34,495 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4653 rows 2024-12-08T00:16:34,496 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T00:16:34,496 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T00:16:34,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:34,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:34,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:34,498 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T00:16:34,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T00:16:34,505 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616994504"}]},"ts":"1733616994504"} 2024-12-08T00:16:34,507 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T00:16:34,507 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T00:16:34,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T00:16:34,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c03237de284ce3f116a5db8c2dd8a9d8, UNASSIGN}] 2024-12-08T00:16:34,510 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c03237de284ce3f116a5db8c2dd8a9d8, UNASSIGN 2024-12-08T00:16:34,511 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=c03237de284ce3f116a5db8c2dd8a9d8, regionState=CLOSING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:34,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c03237de284ce3f116a5db8c2dd8a9d8, UNASSIGN because future has completed 2024-12-08T00:16:34,512 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T00:16:34,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:16:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T00:16:34,666 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:34,666 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T00:16:34,666 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing c03237de284ce3f116a5db8c2dd8a9d8, disabling compactions & flushes 2024-12-08T00:16:34,667 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:34,667 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:34,667 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. after waiting 0 ms 2024-12-08T00:16:34,667 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:34,667 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing c03237de284ce3f116a5db8c2dd8a9d8 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-08T00:16:34,668 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=A 2024-12-08T00:16:34,668 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:34,668 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=B 2024-12-08T00:16:34,668 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:34,668 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c03237de284ce3f116a5db8c2dd8a9d8, store=C 2024-12-08T00:16:34,668 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:34,674 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/45dfe11222064c80ab5e4e40a2c9c1c1 is 50, key is test_row_1/A:col10/1733616994492/Put/seqid=0 2024-12-08T00:16:34,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742323_1499 (size=9857) 2024-12-08T00:16:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T00:16:35,080 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/45dfe11222064c80ab5e4e40a2c9c1c1 2024-12-08T00:16:35,090 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/cb660979e71a4a9e8517bec11e65f944 is 50, key is test_row_1/B:col10/1733616994492/Put/seqid=0 2024-12-08T00:16:35,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742324_1500 (size=9857) 2024-12-08T00:16:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T00:16:35,494 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/cb660979e71a4a9e8517bec11e65f944 2024-12-08T00:16:35,500 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/03b51b0db63e48d288250ab6489a3185 is 50, key is test_row_1/C:col10/1733616994492/Put/seqid=0 2024-12-08T00:16:35,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742325_1501 (size=9857) 2024-12-08T00:16:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T00:16:35,904 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/03b51b0db63e48d288250ab6489a3185 2024-12-08T00:16:35,908 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/A/45dfe11222064c80ab5e4e40a2c9c1c1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/45dfe11222064c80ab5e4e40a2c9c1c1 2024-12-08T00:16:35,911 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/45dfe11222064c80ab5e4e40a2c9c1c1, entries=100, sequenceid=484, filesize=9.6 K 2024-12-08T00:16:35,912 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/B/cb660979e71a4a9e8517bec11e65f944 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/cb660979e71a4a9e8517bec11e65f944 2024-12-08T00:16:35,915 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/cb660979e71a4a9e8517bec11e65f944, entries=100, sequenceid=484, filesize=9.6 K 2024-12-08T00:16:35,916 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/.tmp/C/03b51b0db63e48d288250ab6489a3185 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03b51b0db63e48d288250ab6489a3185 2024-12-08T00:16:35,920 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03b51b0db63e48d288250ab6489a3185, entries=100, sequenceid=484, filesize=9.6 K 2024-12-08T00:16:35,921 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c03237de284ce3f116a5db8c2dd8a9d8 in 1254ms, sequenceid=484, compaction requested=false 2024-12-08T00:16:35,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c7eddb00b48947889a28084d24f959eb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/10df33ec03cb4d1b9539a3c68fd16987, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/7d9874938c9b4650b334ed0fd052ce9d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/bf807e8caae14c8d83b54c4248ab1fb9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/12e179f29d5d4f1d851f033627bd181e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a82a631a13e94be38df81fb628087bf1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/68c6fd49ef534be59b76fdd182f5c58f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81e5c1b902ab4d5a9fcff4e62f034eb7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/df327077af6442e8861d88047e00edbe, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ca5726e99be8401f8f08af2365525009, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/11b53de74c5645a2a98e40e3ca1b2f71, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/226c2aacfe774dc8af9cf8ebc97a6d2d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/eaea035b51c04dd2ba3290b39c23b11d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/1ead373bf038431cb6ebcdf6e4812c15, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/5583cdbc24f6473c8059853a95818afd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/82b8f36b6b8849b689fbaa110bb034b5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ad1141f43aec4ef2959054ab2d818390, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/06e3288735e5403ebe5e0b839b8bb2bd, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/d9390969f29143c08adfd79dd149a7a3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/33e49f4be3b84695a50d71c23e9b16f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/abfa2919fd7a4b39a54f13a9efb106d6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c2b4f023249941a1aece0a6cb2b28916, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81734082bc124f99bc27d4069335ca52, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a319262f7e87454e84d0c5e4ed962583, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/f9b0df9eba6544439bb8ef10c72c08fb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/753bba5fe84b4fa08283b4fad437a45b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/084e5b9e6d2146bd9a831523481a8ed1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/0a5830db66844f85a5da134b19573f46, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/9df19fc5dce74f608e6ee16754b9964c] to archive 2024-12-08T00:16:35,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:16:35,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c7eddb00b48947889a28084d24f959eb to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c7eddb00b48947889a28084d24f959eb 2024-12-08T00:16:35,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/10df33ec03cb4d1b9539a3c68fd16987 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/10df33ec03cb4d1b9539a3c68fd16987 2024-12-08T00:16:35,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/7d9874938c9b4650b334ed0fd052ce9d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/7d9874938c9b4650b334ed0fd052ce9d 2024-12-08T00:16:35,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/bf807e8caae14c8d83b54c4248ab1fb9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/bf807e8caae14c8d83b54c4248ab1fb9 2024-12-08T00:16:35,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/12e179f29d5d4f1d851f033627bd181e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/12e179f29d5d4f1d851f033627bd181e 2024-12-08T00:16:35,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a82a631a13e94be38df81fb628087bf1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a82a631a13e94be38df81fb628087bf1 2024-12-08T00:16:35,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/68c6fd49ef534be59b76fdd182f5c58f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/68c6fd49ef534be59b76fdd182f5c58f 2024-12-08T00:16:35,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81e5c1b902ab4d5a9fcff4e62f034eb7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81e5c1b902ab4d5a9fcff4e62f034eb7 2024-12-08T00:16:35,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/df327077af6442e8861d88047e00edbe to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/df327077af6442e8861d88047e00edbe 2024-12-08T00:16:35,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ca5726e99be8401f8f08af2365525009 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ca5726e99be8401f8f08af2365525009 2024-12-08T00:16:35,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/11b53de74c5645a2a98e40e3ca1b2f71 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/11b53de74c5645a2a98e40e3ca1b2f71 2024-12-08T00:16:35,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/226c2aacfe774dc8af9cf8ebc97a6d2d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/226c2aacfe774dc8af9cf8ebc97a6d2d 2024-12-08T00:16:35,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/eaea035b51c04dd2ba3290b39c23b11d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/eaea035b51c04dd2ba3290b39c23b11d 2024-12-08T00:16:35,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/1ead373bf038431cb6ebcdf6e4812c15 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/1ead373bf038431cb6ebcdf6e4812c15 2024-12-08T00:16:35,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/5583cdbc24f6473c8059853a95818afd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/5583cdbc24f6473c8059853a95818afd 2024-12-08T00:16:35,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/82b8f36b6b8849b689fbaa110bb034b5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/82b8f36b6b8849b689fbaa110bb034b5 2024-12-08T00:16:35,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ad1141f43aec4ef2959054ab2d818390 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/ad1141f43aec4ef2959054ab2d818390 2024-12-08T00:16:35,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/06e3288735e5403ebe5e0b839b8bb2bd to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/06e3288735e5403ebe5e0b839b8bb2bd 2024-12-08T00:16:35,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/d9390969f29143c08adfd79dd149a7a3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/d9390969f29143c08adfd79dd149a7a3 2024-12-08T00:16:35,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/33e49f4be3b84695a50d71c23e9b16f7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/33e49f4be3b84695a50d71c23e9b16f7 2024-12-08T00:16:35,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/abfa2919fd7a4b39a54f13a9efb106d6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/abfa2919fd7a4b39a54f13a9efb106d6 2024-12-08T00:16:35,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c2b4f023249941a1aece0a6cb2b28916 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/c2b4f023249941a1aece0a6cb2b28916 2024-12-08T00:16:35,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81734082bc124f99bc27d4069335ca52 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/81734082bc124f99bc27d4069335ca52 2024-12-08T00:16:35,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a319262f7e87454e84d0c5e4ed962583 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/a319262f7e87454e84d0c5e4ed962583 2024-12-08T00:16:35,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/f9b0df9eba6544439bb8ef10c72c08fb to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/f9b0df9eba6544439bb8ef10c72c08fb 2024-12-08T00:16:35,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/753bba5fe84b4fa08283b4fad437a45b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/753bba5fe84b4fa08283b4fad437a45b 2024-12-08T00:16:35,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/084e5b9e6d2146bd9a831523481a8ed1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/084e5b9e6d2146bd9a831523481a8ed1 2024-12-08T00:16:35,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/0a5830db66844f85a5da134b19573f46 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/0a5830db66844f85a5da134b19573f46 2024-12-08T00:16:35,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/9df19fc5dce74f608e6ee16754b9964c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/9df19fc5dce74f608e6ee16754b9964c 2024-12-08T00:16:35,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/c8ca2b4a2cad4576916bb585980cff55, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/4178b8f0a57e40fabc6ca04e7372d842, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8254e5e9e3c740be97a773902748b9fc, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/911436bf40164f82b7b9f2c9d92d7305, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2573ac9ed505472b93a726abda9d9e54, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6674d50440a242bab29553a6ea600c37, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/04ed9ed9e4df4d6a967edeabdf94143c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b08767fc5f444c9eb52634ddd8796998, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/cf2e8464bc2041f594c999f47e162561, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/fe0739da1720444b8225857bb40accaa, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/241a749e0f9847198350d52ccd3cccda, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/dd04d02effdb48d88d3ae13346d37798, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/3808e2bf214941e5a70116a2a311e9a9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0d16fc0e68c1448eac844ea6753c4617, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/eea9efac62614682b5c34fb5fc81964e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/53cd3e120484451aa838f34d59f4eaaf, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/370851bcf2a24c0db3603bbf48f7541a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b4574edc733e4275bb1e363b2b21035f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8ef41cc4e52e423fb32047964c797e79, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2d1a52522da64a79a88feefb492152ce, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/88d4f353a6184150b8c2c79ceb519d7b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0b385178472c4efdb7e54be87ad56ec7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/5d29cd4cfe62477aa55e0c9a1f05340a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/966487af894c435cbeaae7e6d39df2f9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/712634942473453f9037a7b27bb228e0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/17988a5b77144911b415440859dadffa, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6fb8d0acb4f64ea2a135a8e2ec66d26a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/13e0869c59f54d318069b2f450826b47, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/960c6251ef6f43938980c02559f74156] to archive 2024-12-08T00:16:35,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:16:35,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/c8ca2b4a2cad4576916bb585980cff55 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/c8ca2b4a2cad4576916bb585980cff55 2024-12-08T00:16:35,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/4178b8f0a57e40fabc6ca04e7372d842 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/4178b8f0a57e40fabc6ca04e7372d842 2024-12-08T00:16:35,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8254e5e9e3c740be97a773902748b9fc to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8254e5e9e3c740be97a773902748b9fc 2024-12-08T00:16:35,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/911436bf40164f82b7b9f2c9d92d7305 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/911436bf40164f82b7b9f2c9d92d7305 2024-12-08T00:16:35,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2573ac9ed505472b93a726abda9d9e54 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2573ac9ed505472b93a726abda9d9e54 2024-12-08T00:16:35,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6674d50440a242bab29553a6ea600c37 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6674d50440a242bab29553a6ea600c37 2024-12-08T00:16:35,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/04ed9ed9e4df4d6a967edeabdf94143c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/04ed9ed9e4df4d6a967edeabdf94143c 2024-12-08T00:16:35,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b08767fc5f444c9eb52634ddd8796998 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b08767fc5f444c9eb52634ddd8796998 2024-12-08T00:16:35,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/cf2e8464bc2041f594c999f47e162561 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/cf2e8464bc2041f594c999f47e162561 2024-12-08T00:16:35,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/fe0739da1720444b8225857bb40accaa to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/fe0739da1720444b8225857bb40accaa 2024-12-08T00:16:35,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/241a749e0f9847198350d52ccd3cccda to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/241a749e0f9847198350d52ccd3cccda 2024-12-08T00:16:35,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/dd04d02effdb48d88d3ae13346d37798 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/dd04d02effdb48d88d3ae13346d37798 2024-12-08T00:16:35,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/3808e2bf214941e5a70116a2a311e9a9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/3808e2bf214941e5a70116a2a311e9a9 2024-12-08T00:16:35,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0d16fc0e68c1448eac844ea6753c4617 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0d16fc0e68c1448eac844ea6753c4617 2024-12-08T00:16:35,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/eea9efac62614682b5c34fb5fc81964e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/eea9efac62614682b5c34fb5fc81964e 2024-12-08T00:16:35,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/53cd3e120484451aa838f34d59f4eaaf to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/53cd3e120484451aa838f34d59f4eaaf 2024-12-08T00:16:35,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/370851bcf2a24c0db3603bbf48f7541a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/370851bcf2a24c0db3603bbf48f7541a 2024-12-08T00:16:35,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b4574edc733e4275bb1e363b2b21035f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/b4574edc733e4275bb1e363b2b21035f 2024-12-08T00:16:35,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8ef41cc4e52e423fb32047964c797e79 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/8ef41cc4e52e423fb32047964c797e79 2024-12-08T00:16:35,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2d1a52522da64a79a88feefb492152ce to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/2d1a52522da64a79a88feefb492152ce 2024-12-08T00:16:35,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/88d4f353a6184150b8c2c79ceb519d7b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/88d4f353a6184150b8c2c79ceb519d7b 2024-12-08T00:16:35,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0b385178472c4efdb7e54be87ad56ec7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/0b385178472c4efdb7e54be87ad56ec7 2024-12-08T00:16:35,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/5d29cd4cfe62477aa55e0c9a1f05340a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/5d29cd4cfe62477aa55e0c9a1f05340a 2024-12-08T00:16:35,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/966487af894c435cbeaae7e6d39df2f9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/966487af894c435cbeaae7e6d39df2f9 2024-12-08T00:16:35,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/712634942473453f9037a7b27bb228e0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/712634942473453f9037a7b27bb228e0 2024-12-08T00:16:35,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/17988a5b77144911b415440859dadffa to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/17988a5b77144911b415440859dadffa 2024-12-08T00:16:35,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6fb8d0acb4f64ea2a135a8e2ec66d26a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/6fb8d0acb4f64ea2a135a8e2ec66d26a 2024-12-08T00:16:35,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/13e0869c59f54d318069b2f450826b47 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/13e0869c59f54d318069b2f450826b47 2024-12-08T00:16:35,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/960c6251ef6f43938980c02559f74156 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/960c6251ef6f43938980c02559f74156 2024-12-08T00:16:35,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0c86dc442a60424c8f3f57fa556babe4, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/685218372f714bd8bf6ad3d22ba381d5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/5724c7c6a9454cea86b29376d8872b6f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/439f4c11f7e941968f1ae82bb7db5404, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0facd1babdcf4170959056e375a99b8d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e11b555cd22142ae90bfdcb39de5ea0f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/91e4337e214b4292a6f73ed2270678db, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88f6c014ad8b41cfa925085abd4e555c, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/63762f72a5ff4f5da7338f41384f5cf5, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/9b0d50a7b3f8465fb0c128eb0cdc65ac, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/f48812e699e04ed5beaacb89984a5724, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a9bbe5b4801840819ab2bf9f9da57f07, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03a1dff10ddf4164867bc702e837488d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e3771857aaec47cb9f7ef3101820da2e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/c6d292d789cd4c9ba430827710d281a0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a7e15d7f62c74c8a9e3eafccf30d1ce1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/decd78f34fc1422fb3711bd9aa65820d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/bbb74647bd3b48538d33c99a290395b0, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7357ba81210848209451eeb2da17e085, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/fe8e3a28117746f5ab7f6642f7ba1e39, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/3b3aec1691754164a757a1ff48d97651, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/4ade9b0e3ac84344aa608fedecddd668, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/53f80c9a1269459eb1d7e60200c58c94, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7210e7bec8a348afade70b7f300db675, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/706e6be87a344e6f911350657c60d680, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/59252f0a19244257b16f8a7b7c402000, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/68d1317ad176480480b997d4790f0ace, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/d6b1a338ce124ea6b9054a8bcd288499, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88b2038363e34a1f9bf3ce1cf00b0324] to archive 2024-12-08T00:16:35,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:16:35,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0c86dc442a60424c8f3f57fa556babe4 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0c86dc442a60424c8f3f57fa556babe4 2024-12-08T00:16:35,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/685218372f714bd8bf6ad3d22ba381d5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/685218372f714bd8bf6ad3d22ba381d5 2024-12-08T00:16:35,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/5724c7c6a9454cea86b29376d8872b6f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/5724c7c6a9454cea86b29376d8872b6f 2024-12-08T00:16:35,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/439f4c11f7e941968f1ae82bb7db5404 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/439f4c11f7e941968f1ae82bb7db5404 2024-12-08T00:16:35,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0facd1babdcf4170959056e375a99b8d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/0facd1babdcf4170959056e375a99b8d 2024-12-08T00:16:35,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e11b555cd22142ae90bfdcb39de5ea0f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e11b555cd22142ae90bfdcb39de5ea0f 2024-12-08T00:16:35,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/91e4337e214b4292a6f73ed2270678db to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/91e4337e214b4292a6f73ed2270678db 2024-12-08T00:16:35,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88f6c014ad8b41cfa925085abd4e555c to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88f6c014ad8b41cfa925085abd4e555c 2024-12-08T00:16:35,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/63762f72a5ff4f5da7338f41384f5cf5 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/63762f72a5ff4f5da7338f41384f5cf5 2024-12-08T00:16:35,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/9b0d50a7b3f8465fb0c128eb0cdc65ac to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/9b0d50a7b3f8465fb0c128eb0cdc65ac 2024-12-08T00:16:35,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/f48812e699e04ed5beaacb89984a5724 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/f48812e699e04ed5beaacb89984a5724 2024-12-08T00:16:35,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a9bbe5b4801840819ab2bf9f9da57f07 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a9bbe5b4801840819ab2bf9f9da57f07 2024-12-08T00:16:35,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03a1dff10ddf4164867bc702e837488d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03a1dff10ddf4164867bc702e837488d 2024-12-08T00:16:35,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e3771857aaec47cb9f7ef3101820da2e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/e3771857aaec47cb9f7ef3101820da2e 2024-12-08T00:16:35,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/c6d292d789cd4c9ba430827710d281a0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/c6d292d789cd4c9ba430827710d281a0 2024-12-08T00:16:35,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a7e15d7f62c74c8a9e3eafccf30d1ce1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/a7e15d7f62c74c8a9e3eafccf30d1ce1 2024-12-08T00:16:35,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/decd78f34fc1422fb3711bd9aa65820d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/decd78f34fc1422fb3711bd9aa65820d 2024-12-08T00:16:35,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/bbb74647bd3b48538d33c99a290395b0 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/bbb74647bd3b48538d33c99a290395b0 2024-12-08T00:16:35,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7357ba81210848209451eeb2da17e085 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7357ba81210848209451eeb2da17e085 2024-12-08T00:16:35,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/fe8e3a28117746f5ab7f6642f7ba1e39 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/fe8e3a28117746f5ab7f6642f7ba1e39 2024-12-08T00:16:35,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/3b3aec1691754164a757a1ff48d97651 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/3b3aec1691754164a757a1ff48d97651 2024-12-08T00:16:35,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/4ade9b0e3ac84344aa608fedecddd668 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/4ade9b0e3ac84344aa608fedecddd668 2024-12-08T00:16:35,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/53f80c9a1269459eb1d7e60200c58c94 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/53f80c9a1269459eb1d7e60200c58c94 2024-12-08T00:16:35,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7210e7bec8a348afade70b7f300db675 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/7210e7bec8a348afade70b7f300db675 2024-12-08T00:16:35,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/706e6be87a344e6f911350657c60d680 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/706e6be87a344e6f911350657c60d680 2024-12-08T00:16:35,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/59252f0a19244257b16f8a7b7c402000 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/59252f0a19244257b16f8a7b7c402000 2024-12-08T00:16:35,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/68d1317ad176480480b997d4790f0ace to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/68d1317ad176480480b997d4790f0ace 2024-12-08T00:16:36,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/d6b1a338ce124ea6b9054a8bcd288499 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/d6b1a338ce124ea6b9054a8bcd288499 2024-12-08T00:16:36,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88b2038363e34a1f9bf3ce1cf00b0324 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/88b2038363e34a1f9bf3ce1cf00b0324 2024-12-08T00:16:36,006 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/recovered.edits/487.seqid, newMaxSeqId=487, maxSeqId=1 2024-12-08T00:16:36,006 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. 2024-12-08T00:16:36,006 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for c03237de284ce3f116a5db8c2dd8a9d8: Waiting for close lock at 1733616994666Running coprocessor pre-close hooks at 1733616994666Disabling compacts and flushes for region at 1733616994666Disabling writes for close at 1733616994667 (+1 ms)Obtaining lock to block concurrent updates at 1733616994667Preparing flush snapshotting stores in c03237de284ce3f116a5db8c2dd8a9d8 at 1733616994667Finished memstore snapshotting TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733616994668 (+1 ms)Flushing stores of TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8. at 1733616994669 (+1 ms)Flushing c03237de284ce3f116a5db8c2dd8a9d8/A: creating writer at 1733616994669Flushing c03237de284ce3f116a5db8c2dd8a9d8/A: appending metadata at 1733616994673 (+4 ms)Flushing c03237de284ce3f116a5db8c2dd8a9d8/A: closing flushed file at 1733616994673Flushing c03237de284ce3f116a5db8c2dd8a9d8/B: creating writer at 1733616995087 (+414 ms)Flushing c03237de284ce3f116a5db8c2dd8a9d8/B: appending metadata at 1733616995090 (+3 ms)Flushing c03237de284ce3f116a5db8c2dd8a9d8/B: closing flushed file at 1733616995090Flushing c03237de284ce3f116a5db8c2dd8a9d8/C: creating writer at 1733616995497 (+407 ms)Flushing c03237de284ce3f116a5db8c2dd8a9d8/C: appending metadata at 1733616995499 (+2 ms)Flushing c03237de284ce3f116a5db8c2dd8a9d8/C: closing flushed file at 1733616995499Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9b76714: reopening flushed file at 1733616995907 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d5d5bc5: reopening flushed file at 1733616995911 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37004edf: reopening flushed file at 1733616995915 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c03237de284ce3f116a5db8c2dd8a9d8 in 1254ms, sequenceid=484, compaction requested=false at 1733616995921 (+6 ms)Writing region close event to WAL at 1733616996003 (+82 ms)Running coprocessor post-close hooks at 1733616996006 (+3 ms)Closed at 1733616996006 2024-12-08T00:16:36,008 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:36,009 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=c03237de284ce3f116a5db8c2dd8a9d8, regionState=CLOSED 2024-12-08T00:16:36,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:16:36,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-08T00:16:36,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure c03237de284ce3f116a5db8c2dd8a9d8, server=c1a8cf8d4cfa,36021,1733616847970 in 1.4990 sec 2024-12-08T00:16:36,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-08T00:16:36,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c03237de284ce3f116a5db8c2dd8a9d8, UNASSIGN in 1.5040 sec 2024-12-08T00:16:36,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-08T00:16:36,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5070 sec 2024-12-08T00:16:36,018 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616996018"}]},"ts":"1733616996018"} 2024-12-08T00:16:36,019 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T00:16:36,019 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T00:16:36,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5220 sec 2024-12-08T00:16:36,243 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T00:16:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T00:16:36,651 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:36,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T00:16:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:36,654 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-08T00:16:36,654 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:36,656 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:36,658 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/recovered.edits] 2024-12-08T00:16:36,660 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/45dfe11222064c80ab5e4e40a2c9c1c1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/45dfe11222064c80ab5e4e40a2c9c1c1 2024-12-08T00:16:36,661 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/dc577eb85bab4d23a63eee41474451e1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/A/dc577eb85bab4d23a63eee41474451e1 2024-12-08T00:16:36,663 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/925e0b03a5bd4b08ad1bf7b4aca370ef to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/925e0b03a5bd4b08ad1bf7b4aca370ef 2024-12-08T00:16:36,663 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/cb660979e71a4a9e8517bec11e65f944 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/B/cb660979e71a4a9e8517bec11e65f944 2024-12-08T00:16:36,665 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03b51b0db63e48d288250ab6489a3185 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/03b51b0db63e48d288250ab6489a3185 2024-12-08T00:16:36,666 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/94cc614dd6804b0f88f913a0359192fe to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/C/94cc614dd6804b0f88f913a0359192fe 2024-12-08T00:16:36,669 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/recovered.edits/487.seqid to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8/recovered.edits/487.seqid 2024-12-08T00:16:36,669 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/c03237de284ce3f116a5db8c2dd8a9d8 2024-12-08T00:16:36,669 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T00:16:36,672 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:36,673 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T00:16:36,676 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T00:16:36,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:36,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T00:16:36,677 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733616996677"}]},"ts":"9223372036854775807"} 2024-12-08T00:16:36,680 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T00:16:36,680 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c03237de284ce3f116a5db8c2dd8a9d8, NAME => 'TestAcidGuarantees,,1733616967512.c03237de284ce3f116a5db8c2dd8a9d8.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T00:16:36,680 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T00:16:36,680 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733616996680"}]},"ts":"9223372036854775807"} 2024-12-08T00:16:36,682 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T00:16:36,683 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:36,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-12-08T00:16:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-08T00:16:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T00:16:36,763 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:36,769 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 251) - Thread LEAK? -, OpenFileDescriptor=459 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=406 (was 429), ProcessCount=11 (was 11), AvailableMemoryMB=8928 (was 9066) 2024-12-08T00:16:36,781 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=406, ProcessCount=11, AvailableMemoryMB=8927 2024-12-08T00:16:36,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T00:16:36,782 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:16:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:36,783 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T00:16:36,784 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:36,784 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-12-08T00:16:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T00:16:36,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T00:16:36,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742326_1502 (size=1061) 2024-12-08T00:16:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T00:16:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T00:16:37,192 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329 2024-12-08T00:16:37,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742327_1503 (size=53) 2024-12-08T00:16:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T00:16:37,575 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T00:16:37,597 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:16:37,597 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 951f2f04d16f20e5ba0261e33c9a4276, disabling compactions & flushes 2024-12-08T00:16:37,597 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:37,597 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:37,597 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. after waiting 0 ms 2024-12-08T00:16:37,597 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:37,597 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:37,597 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 951f2f04d16f20e5ba0261e33c9a4276: Waiting for close lock at 1733616997597Disabling compacts and flushes for region at 1733616997597Disabling writes for close at 1733616997597Writing region close event to WAL at 1733616997597Closed at 1733616997597 2024-12-08T00:16:37,598 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T00:16:37,598 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733616997598"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733616997598"}]},"ts":"1733616997598"} 2024-12-08T00:16:37,599 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T00:16:37,600 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T00:16:37,600 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616997600"}]},"ts":"1733616997600"} 2024-12-08T00:16:37,601 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T00:16:37,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, ASSIGN}] 2024-12-08T00:16:37,602 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, ASSIGN 2024-12-08T00:16:37,603 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, ASSIGN; state=OFFLINE, location=c1a8cf8d4cfa,36021,1733616847970; forceNewPlan=false, retain=false 2024-12-08T00:16:37,753 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=951f2f04d16f20e5ba0261e33c9a4276, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:37,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, ASSIGN because future has completed 2024-12-08T00:16:37,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:16:37,912 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:37,912 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:16:37,913 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,913 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:16:37,913 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,913 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,915 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,917 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:16:37,917 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951f2f04d16f20e5ba0261e33c9a4276 columnFamilyName A 2024-12-08T00:16:37,917 DEBUG [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:37,918 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(327): Store=951f2f04d16f20e5ba0261e33c9a4276/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:16:37,918 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,919 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:16:37,919 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951f2f04d16f20e5ba0261e33c9a4276 columnFamilyName B 2024-12-08T00:16:37,919 DEBUG [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:37,920 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(327): Store=951f2f04d16f20e5ba0261e33c9a4276/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:16:37,920 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T00:16:37,921 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:16:37,921 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951f2f04d16f20e5ba0261e33c9a4276 columnFamilyName C 2024-12-08T00:16:37,921 DEBUG [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:37,921 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(327): Store=951f2f04d16f20e5ba0261e33c9a4276/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:16:37,921 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:37,922 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,922 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,923 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,924 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,924 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,925 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:16:37,926 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,928 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T00:16:37,929 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 951f2f04d16f20e5ba0261e33c9a4276; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73403953, jitterRate=0.09380413591861725}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:16:37,929 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:37,930 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 951f2f04d16f20e5ba0261e33c9a4276: Running coprocessor pre-open hook at 1733616997914Writing region info on filesystem at 1733616997914Initializing all the Stores at 1733616997915 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616997915Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616997915Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616997915Cleaning up temporary data from old regions at 1733616997924 (+9 ms)Running coprocessor post-open hooks at 1733616997929 (+5 ms)Region opened successfully at 1733616997930 (+1 ms) 2024-12-08T00:16:37,930 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., pid=146, masterSystemTime=1733616997906 2024-12-08T00:16:37,932 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:37,933 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:37,933 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=951f2f04d16f20e5ba0261e33c9a4276, regionState=OPEN, openSeqNum=2, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:37,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:16:37,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-08T00:16:37,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 in 181 msec 2024-12-08T00:16:37,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-08T00:16:37,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, ASSIGN in 336 msec 2024-12-08T00:16:37,940 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T00:16:37,940 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733616997940"}]},"ts":"1733616997940"} 2024-12-08T00:16:37,941 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T00:16:37,942 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T00:16:37,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2024-12-08T00:16:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T00:16:38,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:38,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:38,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:38,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:38,937 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7790c422, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:38,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1064125e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:38,941 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:38,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:38,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T00:16:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:16:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b1732be 2024-12-08T00:16:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T00:16:38,947 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T00:16:38,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T00:16:38,949 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T00:16:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T00:16:38,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742328_1504 (size=1097) 2024-12-08T00:16:39,365 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T00:16:39,365 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T00:16:39,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T00:16:39,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, REOPEN/MOVE}] 2024-12-08T00:16:39,374 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, REOPEN/MOVE 2024-12-08T00:16:39,376 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=951f2f04d16f20e5ba0261e33c9a4276, regionState=CLOSING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:39,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, REOPEN/MOVE because future has completed 2024-12-08T00:16:39,378 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T00:16:39,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:16:39,532 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,532 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T00:16:39,532 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 951f2f04d16f20e5ba0261e33c9a4276, disabling compactions & flushes 2024-12-08T00:16:39,532 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:39,532 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:39,532 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. after waiting 0 ms 2024-12-08T00:16:39,532 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:39,539 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T00:16:39,540 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:39,540 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 951f2f04d16f20e5ba0261e33c9a4276: Waiting for close lock at 1733616999532Running coprocessor pre-close hooks at 1733616999532Disabling compacts and flushes for region at 1733616999532Disabling writes for close at 1733616999532Writing region close event to WAL at 1733616999533 (+1 ms)Running coprocessor post-close hooks at 1733616999539 (+6 ms)Closed at 1733616999540 (+1 ms) 2024-12-08T00:16:39,540 WARN [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 951f2f04d16f20e5ba0261e33c9a4276 to self. 2024-12-08T00:16:39,543 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,543 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=951f2f04d16f20e5ba0261e33c9a4276, regionState=CLOSED 2024-12-08T00:16:39,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:16:39,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-08T00:16:39,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 in 168 msec 2024-12-08T00:16:39,549 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, REOPEN/MOVE; state=CLOSED, location=c1a8cf8d4cfa,36021,1733616847970; forceNewPlan=false, retain=true 2024-12-08T00:16:39,700 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=951f2f04d16f20e5ba0261e33c9a4276, regionState=OPENING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:39,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, REOPEN/MOVE because future has completed 2024-12-08T00:16:39,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:16:39,863 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:39,864 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} 2024-12-08T00:16:39,864 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,864 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T00:16:39,864 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,864 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,866 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,867 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:16:39,867 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951f2f04d16f20e5ba0261e33c9a4276 columnFamilyName A 2024-12-08T00:16:39,868 DEBUG [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:39,869 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(327): Store=951f2f04d16f20e5ba0261e33c9a4276/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:16:39,869 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,869 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:16:39,870 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951f2f04d16f20e5ba0261e33c9a4276 columnFamilyName B 2024-12-08T00:16:39,870 DEBUG [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:39,870 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(327): Store=951f2f04d16f20e5ba0261e33c9a4276/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:16:39,870 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,871 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T00:16:39,871 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951f2f04d16f20e5ba0261e33c9a4276 columnFamilyName C 2024-12-08T00:16:39,871 DEBUG [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:39,871 INFO [StoreOpener-951f2f04d16f20e5ba0261e33c9a4276-1 {}] regionserver.HStore(327): Store=951f2f04d16f20e5ba0261e33c9a4276/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T00:16:39,871 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:39,871 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,872 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,873 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,874 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,874 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,874 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T00:16:39,875 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,876 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 951f2f04d16f20e5ba0261e33c9a4276; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62911891, jitterRate=-0.06253977119922638}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T00:16:39,876 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,876 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 951f2f04d16f20e5ba0261e33c9a4276: Running coprocessor pre-open hook at 1733616999864Writing region info on filesystem at 1733616999864Initializing all the Stores at 1733616999865 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616999865Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616999866 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733616999866Cleaning up temporary data from old regions at 1733616999874 (+8 ms)Running coprocessor post-open hooks at 1733616999876 (+2 ms)Region opened successfully at 1733616999876 2024-12-08T00:16:39,877 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., pid=151, masterSystemTime=1733616999856 2024-12-08T00:16:39,878 DEBUG [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:39,878 INFO [RS_OPEN_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:39,879 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=951f2f04d16f20e5ba0261e33c9a4276, regionState=OPEN, openSeqNum=5, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:39,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:16:39,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-12-08T00:16:39,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 in 178 msec 2024-12-08T00:16:39,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-08T00:16:39,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, REOPEN/MOVE in 510 msec 2024-12-08T00:16:39,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-08T00:16:39,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-12-08T00:16:39,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-12-08T00:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-08T00:16:39,891 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:39,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,893 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d6ba34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,895 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e00bd1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,897 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acff498, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,899 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,901 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45abb74b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c29346f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2cbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,907 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379c8cbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,910 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2282e51e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,911 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e829d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,914 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51778469, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,915 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffd46de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,917 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181f2033, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,919 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40797132, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,921 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db16348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,922 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135b0315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,923 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66800acc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,924 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e7ba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c1a8cf8d4cfa,46259,-1 for getting cluster id 2024-12-08T00:16:39,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T00:16:39,926 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '360fcdd9-b471-4f97-a388-64d212b7bcc8' 2024-12-08T00:16:39,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T00:16:39,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "360fcdd9-b471-4f97-a388-64d212b7bcc8" 2024-12-08T00:16:39,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7798ef82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c1a8cf8d4cfa,46259,-1] 2024-12-08T00:16:39,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T00:16:39,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:39,927 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T00:16:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6ff995, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T00:16:39,928 DEBUG [Thread-2275 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,928 DEBUG [Thread-2271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,928 DEBUG [Thread-2269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,928 DEBUG [Thread-2260 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:39,928 DEBUG [Thread-2267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,928 DEBUG [Thread-2264 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,928 DEBUG [Thread-2256 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-08T00:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,929 DEBUG [Thread-2262 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,929 DEBUG [Thread-2258 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,929 DEBUG [Thread-2273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T00:16:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T00:16:39,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=-1] 2024-12-08T00:16:39,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T00:16:39,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T00:16:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,935 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5] 2024-12-08T00:16:39,937 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:39,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:16:39,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:39,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:39,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:39,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:39,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:39,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:39,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617059958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:39,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733616999949, max timestamp=1733616999961Num uniques 859; 2024-12-08T00:16:39,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733616999949, max timestamp=1733616999961Num uniques 859; 2024-12-08T00:16:39,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T00:16:39,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dbc5f93143294853b295058853a56172_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733616999941/Put/seqid=0 2024-12-08T00:16:39,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733616999949, max timestamp=1733616999961Num uniques 859; 2024-12-08T00:16:39,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T00:16:39,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T00:16:39,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742329_1505 (size=12154) 2024-12-08T00:16:39,978 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:39,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dbc5f93143294853b295058853a56172_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dbc5f93143294853b295058853a56172_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:39,982 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/735b202a01cf440b92bde89a23342c88, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:39,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/735b202a01cf440b92bde89a23342c88 is 175, key is test_row_0/A:col10/1733616999941/Put/seqid=0 2024-12-08T00:16:39,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617059979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:39,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742330_1506 (size=30955) 2024-12-08T00:16:39,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617059982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:39,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617059983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:39,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617059983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T00:16:40,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617060071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T00:16:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617060091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617060091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617060092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617060092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T00:16:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T00:16:40,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617060281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617060300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617060300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617060301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617060301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,386 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/735b202a01cf440b92bde89a23342c88 2024-12-08T00:16:40,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T00:16:40,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:40,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/a5eebd7f5bad4ac286dfeef44e3f44a2 is 50, key is test_row_0/B:col10/1733616999941/Put/seqid=0 2024-12-08T00:16:40,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742331_1507 (size=12001) 2024-12-08T00:16:40,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T00:16:40,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:40,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T00:16:40,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617060591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617060610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617060610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617060610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617060611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:40,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T00:16:40,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/a5eebd7f5bad4ac286dfeef44e3f44a2 2024-12-08T00:16:40,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/d730e278ecf343ec93d2e2796716ceaa is 50, key is test_row_0/C:col10/1733616999941/Put/seqid=0 2024-12-08T00:16:40,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742332_1508 (size=12001) 2024-12-08T00:16:40,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T00:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:40,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:41,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T00:16:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:41,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T00:16:41,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617061101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:41,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617061120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:41,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617061120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617061121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617061121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:41,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:41,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T00:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:41,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:41,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/d730e278ecf343ec93d2e2796716ceaa 2024-12-08T00:16:41,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/735b202a01cf440b92bde89a23342c88 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/735b202a01cf440b92bde89a23342c88 2024-12-08T00:16:41,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/735b202a01cf440b92bde89a23342c88, entries=150, sequenceid=15, filesize=30.2 K 2024-12-08T00:16:41,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/a5eebd7f5bad4ac286dfeef44e3f44a2 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a5eebd7f5bad4ac286dfeef44e3f44a2 2024-12-08T00:16:41,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a5eebd7f5bad4ac286dfeef44e3f44a2, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T00:16:41,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/d730e278ecf343ec93d2e2796716ceaa as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d730e278ecf343ec93d2e2796716ceaa 2024-12-08T00:16:41,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d730e278ecf343ec93d2e2796716ceaa, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T00:16:41,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 951f2f04d16f20e5ba0261e33c9a4276 in 1298ms, sequenceid=15, compaction requested=false 2024-12-08T00:16:41,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:41,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T00:16:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:41,310 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-08T00:16:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:41,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083fd7ee9290b34172aa7b85c01255d3cd_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733616999975/Put/seqid=0 2024-12-08T00:16:41,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742333_1509 (size=12154) 2024-12-08T00:16:41,520 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T00:16:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:41,722 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083fd7ee9290b34172aa7b85c01255d3cd_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083fd7ee9290b34172aa7b85c01255d3cd_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:41,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/2846d10135234f99849050eeee0d4bc1, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:41,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/2846d10135234f99849050eeee0d4bc1 is 175, key is test_row_0/A:col10/1733616999975/Put/seqid=0 2024-12-08T00:16:41,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742334_1510 (size=30955) 2024-12-08T00:16:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T00:16:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:42,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:42,131 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/2846d10135234f99849050eeee0d4bc1 2024-12-08T00:16:42,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617062133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/a47d1124c40046cf8a88d595c63c6205 is 50, key is test_row_0/B:col10/1733616999975/Put/seqid=0 2024-12-08T00:16:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742335_1511 (size=12001) 2024-12-08T00:16:42,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617062136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617062136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617062140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617062141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,144 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,144 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617062240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617062251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617062251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617062451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617062460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617062460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,540 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/a47d1124c40046cf8a88d595c63c6205 2024-12-08T00:16:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/d26a8086bbee447d9eff7b8f8e347b6d is 50, key is test_row_0/C:col10/1733616999975/Put/seqid=0 2024-12-08T00:16:42,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742336_1512 (size=12001) 2024-12-08T00:16:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617062761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617062771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617062771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:42,967 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/d26a8086bbee447d9eff7b8f8e347b6d 2024-12-08T00:16:42,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/2846d10135234f99849050eeee0d4bc1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2846d10135234f99849050eeee0d4bc1 2024-12-08T00:16:42,973 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2846d10135234f99849050eeee0d4bc1, entries=150, sequenceid=49, filesize=30.2 K 2024-12-08T00:16:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/a47d1124c40046cf8a88d595c63c6205 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a47d1124c40046cf8a88d595c63c6205 2024-12-08T00:16:42,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a47d1124c40046cf8a88d595c63c6205, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T00:16:42,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/d26a8086bbee447d9eff7b8f8e347b6d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d26a8086bbee447d9eff7b8f8e347b6d 2024-12-08T00:16:42,979 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d26a8086bbee447d9eff7b8f8e347b6d, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T00:16:42,979 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 951f2f04d16f20e5ba0261e33c9a4276 in 1669ms, sequenceid=49, compaction requested=false 2024-12-08T00:16:42,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:42,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:42,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-08T00:16:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-08T00:16:42,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-08T00:16:42,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0440 sec 2024-12-08T00:16:42,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.0540 sec 2024-12-08T00:16:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:43,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T00:16:43,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:43,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:43,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:43,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:43,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:43,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:43,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733617003291, max timestamp=1733617003300Num uniques 859; 2024-12-08T00:16:43,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T00:16:43,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733617003291, max timestamp=1733617003300Num uniques 859; 2024-12-08T00:16:43,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T00:16:43,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733617003291, max timestamp=1733617003300Num uniques 859; 2024-12-08T00:16:43,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T00:16:43,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208659eebd993ac439ebfe2a375a526c65f_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617002131/Put/seqid=0 2024-12-08T00:16:43,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742337_1513 (size=14594) 2024-12-08T00:16:43,311 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:43,314 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208659eebd993ac439ebfe2a375a526c65f_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208659eebd993ac439ebfe2a375a526c65f_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:43,315 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/894706054cf749c9ad46f0f8c9fb447d, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:43,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/894706054cf749c9ad46f0f8c9fb447d is 175, key is test_row_0/A:col10/1733617002131/Put/seqid=0 2024-12-08T00:16:43,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742338_1514 (size=39549) 2024-12-08T00:16:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617063322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617063322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617063323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617063431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617063431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617063431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617063641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617063641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617063641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,720 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/894706054cf749c9ad46f0f8c9fb447d 2024-12-08T00:16:43,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/ee6ea77cd47548a490940a7b0910d0e6 is 50, key is test_row_0/B:col10/1733617002131/Put/seqid=0 2024-12-08T00:16:43,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742339_1515 (size=12001) 2024-12-08T00:16:43,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/ee6ea77cd47548a490940a7b0910d0e6 2024-12-08T00:16:43,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/e0cc6fbf3b7541f7850072afe8cf7633 is 50, key is test_row_0/C:col10/1733617002131/Put/seqid=0 2024-12-08T00:16:43,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742340_1516 (size=12001) 2024-12-08T00:16:43,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617063951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617063951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:43,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617063951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T00:16:44,101 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:44,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-08T00:16:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T00:16:44,103 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:44,104 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:44,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:44,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/e0cc6fbf3b7541f7850072afe8cf7633 2024-12-08T00:16:44,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/894706054cf749c9ad46f0f8c9fb447d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/894706054cf749c9ad46f0f8c9fb447d 2024-12-08T00:16:44,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/894706054cf749c9ad46f0f8c9fb447d, entries=200, sequenceid=61, filesize=38.6 K 2024-12-08T00:16:44,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/ee6ea77cd47548a490940a7b0910d0e6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ee6ea77cd47548a490940a7b0910d0e6 2024-12-08T00:16:44,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ee6ea77cd47548a490940a7b0910d0e6, entries=150, sequenceid=61, filesize=11.7 K 2024-12-08T00:16:44,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/e0cc6fbf3b7541f7850072afe8cf7633 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e0cc6fbf3b7541f7850072afe8cf7633 2024-12-08T00:16:44,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e0cc6fbf3b7541f7850072afe8cf7633, entries=150, sequenceid=61, filesize=11.7 K 2024-12-08T00:16:44,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 951f2f04d16f20e5ba0261e33c9a4276 in 882ms, sequenceid=61, compaction requested=true 2024-12-08T00:16:44,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:44,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:44,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:44,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:44,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:44,155 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:44,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:44,155 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:44,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/B is initiating minor compaction (all files) 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/A is initiating minor compaction (all files) 2024-12-08T00:16:44,156 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/B in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,156 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/A in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,156 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a5eebd7f5bad4ac286dfeef44e3f44a2, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a47d1124c40046cf8a88d595c63c6205, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ee6ea77cd47548a490940a7b0910d0e6] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=35.2 K 2024-12-08T00:16:44,156 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/735b202a01cf440b92bde89a23342c88, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2846d10135234f99849050eeee0d4bc1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/894706054cf749c9ad46f0f8c9fb447d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=99.1 K 2024-12-08T00:16:44,156 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/735b202a01cf440b92bde89a23342c88, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2846d10135234f99849050eeee0d4bc1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/894706054cf749c9ad46f0f8c9fb447d] 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a5eebd7f5bad4ac286dfeef44e3f44a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733616999940 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a47d1124c40046cf8a88d595c63c6205, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733616999975 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 735b202a01cf440b92bde89a23342c88, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733616999940 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ee6ea77cd47548a490940a7b0910d0e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733617002131 2024-12-08T00:16:44,156 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2846d10135234f99849050eeee0d4bc1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733616999975 2024-12-08T00:16:44,157 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 894706054cf749c9ad46f0f8c9fb447d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733617002127 2024-12-08T00:16:44,162 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:44,163 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:44,163 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/ca522ffe0d5646c498173fa8c2647b72 is 50, key is test_row_0/B:col10/1733617002131/Put/seqid=0 2024-12-08T00:16:44,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-08T00:16:44,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:44,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:44,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:44,164 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208f37806a23f314fa9a5ec799c2f0e8099_951f2f04d16f20e5ba0261e33c9a4276 store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:44,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:44,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:44,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:44,165 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208f37806a23f314fa9a5ec799c2f0e8099_951f2f04d16f20e5ba0261e33c9a4276, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:44,165 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f37806a23f314fa9a5ec799c2f0e8099_951f2f04d16f20e5ba0261e33c9a4276 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:44,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742341_1517 (size=12104) 2024-12-08T00:16:44,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742342_1518 (size=4469) 2024-12-08T00:16:44,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120847c69d11700f4137befafff0bebc7832_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617003320/Put/seqid=0 2024-12-08T00:16:44,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/ca522ffe0d5646c498173fa8c2647b72 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ca522ffe0d5646c498173fa8c2647b72 2024-12-08T00:16:44,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617064184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,193 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/B of 951f2f04d16f20e5ba0261e33c9a4276 into ca522ffe0d5646c498173fa8c2647b72(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617064189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,193 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:44,193 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/B, priority=13, startTime=1733617004155; duration=0sec 2024-12-08T00:16:44,193 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:44,193 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:B 2024-12-08T00:16:44,193 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,194 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:44,194 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/C is initiating minor compaction (all files) 2024-12-08T00:16:44,194 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/C in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,194 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d730e278ecf343ec93d2e2796716ceaa, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d26a8086bbee447d9eff7b8f8e347b6d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e0cc6fbf3b7541f7850072afe8cf7633] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=35.2 K 2024-12-08T00:16:44,194 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d730e278ecf343ec93d2e2796716ceaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733616999940 2024-12-08T00:16:44,195 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d26a8086bbee447d9eff7b8f8e347b6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733616999975 2024-12-08T00:16:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742343_1519 (size=14594) 2024-12-08T00:16:44,195 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e0cc6fbf3b7541f7850072afe8cf7633, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733617002131 2024-12-08T00:16:44,199 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#C#compaction#438 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:44,200 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/d414af87bc12486a93f524554b362b64 is 50, key is test_row_0/C:col10/1733617002131/Put/seqid=0 2024-12-08T00:16:44,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742344_1520 (size=12104) 2024-12-08T00:16:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T00:16:44,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617064291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617064301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:44,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:44,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T00:16:44,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617064460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617064461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617064461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617064501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617064510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:44,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:44,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,584 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#A#compaction#436 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:44,585 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/233b3d5bf5ee4427b4dfd8bfd50d5e36 is 175, key is test_row_0/A:col10/1733617002131/Put/seqid=0 2024-12-08T00:16:44,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742345_1521 (size=31058) 2024-12-08T00:16:44,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:44,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120847c69d11700f4137befafff0bebc7832_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120847c69d11700f4137befafff0bebc7832_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:44,598 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/73e9c64fd0024a53b695898ff5fdc9ae, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:44,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/73e9c64fd0024a53b695898ff5fdc9ae is 175, key is test_row_0/A:col10/1733617003320/Put/seqid=0 2024-12-08T00:16:44,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742346_1522 (size=39549) 2024-12-08T00:16:44,606 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/d414af87bc12486a93f524554b362b64 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d414af87bc12486a93f524554b362b64 2024-12-08T00:16:44,609 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/C of 951f2f04d16f20e5ba0261e33c9a4276 into d414af87bc12486a93f524554b362b64(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:44,609 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:44,609 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/C, priority=13, startTime=1733617004155; duration=0sec 2024-12-08T00:16:44,609 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:44,609 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:C 2024-12-08T00:16:44,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T00:16:44,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617064811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617064821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:44,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:44,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:44,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:44,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:44,993 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/233b3d5bf5ee4427b4dfd8bfd50d5e36 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/233b3d5bf5ee4427b4dfd8bfd50d5e36 2024-12-08T00:16:44,997 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/A of 951f2f04d16f20e5ba0261e33c9a4276 into 233b3d5bf5ee4427b4dfd8bfd50d5e36(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:44,997 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:44,997 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/A, priority=13, startTime=1733617004155; duration=0sec 2024-12-08T00:16:44,997 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:44,997 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:A 2024-12-08T00:16:45,002 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/73e9c64fd0024a53b695898ff5fdc9ae 2024-12-08T00:16:45,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/0ea97b8a88d244ae8e806db9530b24d6 is 50, key is test_row_0/B:col10/1733617003320/Put/seqid=0 2024-12-08T00:16:45,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742347_1523 (size=12001) 2024-12-08T00:16:45,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:45,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:45,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T00:16:45,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:45,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:45,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617065321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617065331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/0ea97b8a88d244ae8e806db9530b24d6 2024-12-08T00:16:45,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/16061dc79eef44839a9c8b57c212d998 is 50, key is test_row_0/C:col10/1733617003320/Put/seqid=0 2024-12-08T00:16:45,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742348_1524 (size=12001) 2024-12-08T00:16:45,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/16061dc79eef44839a9c8b57c212d998 2024-12-08T00:16:45,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/73e9c64fd0024a53b695898ff5fdc9ae as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/73e9c64fd0024a53b695898ff5fdc9ae 2024-12-08T00:16:45,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/73e9c64fd0024a53b695898ff5fdc9ae, entries=200, sequenceid=95, filesize=38.6 K 2024-12-08T00:16:45,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/0ea97b8a88d244ae8e806db9530b24d6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/0ea97b8a88d244ae8e806db9530b24d6 2024-12-08T00:16:45,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/0ea97b8a88d244ae8e806db9530b24d6, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T00:16:45,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/16061dc79eef44839a9c8b57c212d998 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/16061dc79eef44839a9c8b57c212d998 2024-12-08T00:16:45,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/16061dc79eef44839a9c8b57c212d998, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T00:16:45,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 951f2f04d16f20e5ba0261e33c9a4276 in 1272ms, sequenceid=95, compaction requested=false 2024-12-08T00:16:45,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:45,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T00:16:45,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:45,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:45,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:45,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:45,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:45,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:45,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b2dc2c97df5845e6a6f2557ee6053ea2_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617004165/Put/seqid=0 2024-12-08T00:16:45,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742349_1525 (size=14594) 2024-12-08T00:16:45,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617065501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617065502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617065502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617065610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617065610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617065610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617065821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617065821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617065821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:45,886 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:45,888 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b2dc2c97df5845e6a6f2557ee6053ea2_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b2dc2c97df5845e6a6f2557ee6053ea2_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:45,889 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/27e80f247e084f53999527af03e34f7a, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:45,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/27e80f247e084f53999527af03e34f7a is 175, key is test_row_0/A:col10/1733617004165/Put/seqid=0 2024-12-08T00:16:45,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742350_1526 (size=39549) 2024-12-08T00:16:45,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:45,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:45,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:45,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:46,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617066131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:46,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617066131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:46,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617066131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:46,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:46,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T00:16:46,292 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/27e80f247e084f53999527af03e34f7a 2024-12-08T00:16:46,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/e6691627fcad41f1b0951b5e59284ad4 is 50, key is test_row_0/B:col10/1733617004165/Put/seqid=0 2024-12-08T00:16:46,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742351_1527 (size=12001) 2024-12-08T00:16:46,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617066340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:46,341 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:46,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617066351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:46,353 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:46,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:46,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:46,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617066641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:46,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617066641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:46,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617066641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:46,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:46,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:46,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/e6691627fcad41f1b0951b5e59284ad4 2024-12-08T00:16:46,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/ef73429da54f411bac12c954652701c4 is 50, key is test_row_0/C:col10/1733617004165/Put/seqid=0 2024-12-08T00:16:46,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742352_1528 (size=12001) 2024-12-08T00:16:46,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:46,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:47,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:47,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:47,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:47,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:47,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:47,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/ef73429da54f411bac12c954652701c4 2024-12-08T00:16:47,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/27e80f247e084f53999527af03e34f7a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/27e80f247e084f53999527af03e34f7a 2024-12-08T00:16:47,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/27e80f247e084f53999527af03e34f7a, entries=200, sequenceid=112, filesize=38.6 K 2024-12-08T00:16:47,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/e6691627fcad41f1b0951b5e59284ad4 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e6691627fcad41f1b0951b5e59284ad4 2024-12-08T00:16:47,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e6691627fcad41f1b0951b5e59284ad4, entries=150, sequenceid=112, filesize=11.7 K 2024-12-08T00:16:47,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/ef73429da54f411bac12c954652701c4 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/ef73429da54f411bac12c954652701c4 2024-12-08T00:16:47,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/ef73429da54f411bac12c954652701c4, entries=150, sequenceid=112, filesize=11.7 K 2024-12-08T00:16:47,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 951f2f04d16f20e5ba0261e33c9a4276 in 1662ms, sequenceid=112, compaction requested=true 2024-12-08T00:16:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:47,135 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:47,135 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:47,135 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:47,135 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:47,135 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/A is initiating minor compaction (all files) 2024-12-08T00:16:47,135 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/B is initiating minor compaction (all files) 2024-12-08T00:16:47,135 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/A in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:47,135 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/B in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:47,136 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/233b3d5bf5ee4427b4dfd8bfd50d5e36, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/73e9c64fd0024a53b695898ff5fdc9ae, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/27e80f247e084f53999527af03e34f7a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=107.6 K 2024-12-08T00:16:47,136 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ca522ffe0d5646c498173fa8c2647b72, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/0ea97b8a88d244ae8e806db9530b24d6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e6691627fcad41f1b0951b5e59284ad4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=35.3 K 2024-12-08T00:16:47,136 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:47,136 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/233b3d5bf5ee4427b4dfd8bfd50d5e36, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/73e9c64fd0024a53b695898ff5fdc9ae, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/27e80f247e084f53999527af03e34f7a] 2024-12-08T00:16:47,136 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ca522ffe0d5646c498173fa8c2647b72, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733617002131 2024-12-08T00:16:47,136 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 233b3d5bf5ee4427b4dfd8bfd50d5e36, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733617002131 2024-12-08T00:16:47,136 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73e9c64fd0024a53b695898ff5fdc9ae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733617003315 2024-12-08T00:16:47,136 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ea97b8a88d244ae8e806db9530b24d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733617003315 2024-12-08T00:16:47,137 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e80f247e084f53999527af03e34f7a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733617004165 2024-12-08T00:16:47,137 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e6691627fcad41f1b0951b5e59284ad4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733617004165 2024-12-08T00:16:47,141 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:47,141 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#B#compaction#444 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:47,142 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/5b10977367094da6be81b3c2f22acde1 is 50, key is test_row_0/B:col10/1733617004165/Put/seqid=0 2024-12-08T00:16:47,143 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412089bed2ba14b904384a2facf37092eded3_951f2f04d16f20e5ba0261e33c9a4276 store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:47,144 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412089bed2ba14b904384a2facf37092eded3_951f2f04d16f20e5ba0261e33c9a4276, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:47,144 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089bed2ba14b904384a2facf37092eded3_951f2f04d16f20e5ba0261e33c9a4276 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:47,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T00:16:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:47,153 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T00:16:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:47,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742353_1529 (size=12207) 2024-12-08T00:16:47,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742354_1530 (size=4469) 2024-12-08T00:16:47,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e1409b28e6cf418cb9bd513905df4cd8_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617005501/Put/seqid=0 2024-12-08T00:16:47,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742355_1531 (size=12254) 2024-12-08T00:16:47,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:47,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e1409b28e6cf418cb9bd513905df4cd8_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e1409b28e6cf418cb9bd513905df4cd8_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:47,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/2e719d416e0e4a29b45f8c7b69549942, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:47,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/2e719d416e0e4a29b45f8c7b69549942 is 175, key is test_row_0/A:col10/1733617005501/Put/seqid=0 2024-12-08T00:16:47,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742356_1532 (size=31055) 2024-12-08T00:16:47,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/2e719d416e0e4a29b45f8c7b69549942 2024-12-08T00:16:47,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/49a55d319cda4396a48c88f7e822526d is 50, key is test_row_0/B:col10/1733617005501/Put/seqid=0 2024-12-08T00:16:47,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742357_1533 (size=12101) 2024-12-08T00:16:47,563 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/5b10977367094da6be81b3c2f22acde1 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/5b10977367094da6be81b3c2f22acde1 2024-12-08T00:16:47,567 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/B of 951f2f04d16f20e5ba0261e33c9a4276 into 5b10977367094da6be81b3c2f22acde1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:47,567 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:47,567 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/B, priority=13, startTime=1733617007135; duration=0sec 2024-12-08T00:16:47,567 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:47,567 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:B 2024-12-08T00:16:47,567 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:47,568 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#A#compaction#445 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:47,568 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:47,568 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/913a23e7f329403e8f47d707cace20ff is 175, key is test_row_0/A:col10/1733617004165/Put/seqid=0 2024-12-08T00:16:47,568 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/C is initiating minor compaction (all files) 2024-12-08T00:16:47,568 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/C in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:47,569 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d414af87bc12486a93f524554b362b64, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/16061dc79eef44839a9c8b57c212d998, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/ef73429da54f411bac12c954652701c4] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=35.3 K 2024-12-08T00:16:47,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d414af87bc12486a93f524554b362b64, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733617002131 2024-12-08T00:16:47,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 16061dc79eef44839a9c8b57c212d998, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733617003315 2024-12-08T00:16:47,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting ef73429da54f411bac12c954652701c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733617004165 2024-12-08T00:16:47,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742358_1534 (size=31161) 2024-12-08T00:16:47,575 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T00:16:47,575 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T00:16:47,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/913a23e7f329403e8f47d707cace20ff as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/913a23e7f329403e8f47d707cace20ff 2024-12-08T00:16:47,578 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:47,578 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/00add05daa65473c93dfca57c4f29444 is 50, key is test_row_0/C:col10/1733617004165/Put/seqid=0 2024-12-08T00:16:47,583 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/A of 951f2f04d16f20e5ba0261e33c9a4276 into 913a23e7f329403e8f47d707cace20ff(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:47,583 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:47,583 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/A, priority=13, startTime=1733617007135; duration=0sec 2024-12-08T00:16:47,583 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:47,583 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:A 2024-12-08T00:16:47,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742359_1535 (size=12207) 2024-12-08T00:16:47,592 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/49a55d319cda4396a48c88f7e822526d 2024-12-08T00:16:47,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/c7c65409aa4e4804b8379a019260c96d is 50, key is test_row_0/C:col10/1733617005501/Put/seqid=0 2024-12-08T00:16:47,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742360_1536 (size=12101) 2024-12-08T00:16:47,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:47,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617067673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:47,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:47,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617067674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:47,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:47,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617067674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:47,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617067780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:47,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617067780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:47,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617067780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:47,989 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/00add05daa65473c93dfca57c4f29444 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/00add05daa65473c93dfca57c4f29444 2024-12-08T00:16:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617067991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:47,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617067991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:47,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617067991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:47,994 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/C of 951f2f04d16f20e5ba0261e33c9a4276 into 00add05daa65473c93dfca57c4f29444(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:47,995 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:47,995 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/C, priority=13, startTime=1733617007135; duration=0sec 2024-12-08T00:16:47,995 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:47,995 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:C 2024-12-08T00:16:48,006 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/c7c65409aa4e4804b8379a019260c96d 2024-12-08T00:16:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/2e719d416e0e4a29b45f8c7b69549942 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2e719d416e0e4a29b45f8c7b69549942 2024-12-08T00:16:48,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2e719d416e0e4a29b45f8c7b69549942, entries=150, sequenceid=131, filesize=30.3 K 2024-12-08T00:16:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/49a55d319cda4396a48c88f7e822526d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/49a55d319cda4396a48c88f7e822526d 2024-12-08T00:16:48,016 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/49a55d319cda4396a48c88f7e822526d, entries=150, sequenceid=131, filesize=11.8 K 2024-12-08T00:16:48,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/c7c65409aa4e4804b8379a019260c96d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/c7c65409aa4e4804b8379a019260c96d 2024-12-08T00:16:48,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/c7c65409aa4e4804b8379a019260c96d, entries=150, sequenceid=131, filesize=11.8 K 2024-12-08T00:16:48,024 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 951f2f04d16f20e5ba0261e33c9a4276 in 871ms, sequenceid=131, compaction requested=false 2024-12-08T00:16:48,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:48,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:48,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-08T00:16:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-08T00:16:48,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-08T00:16:48,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9210 sec 2024-12-08T00:16:48,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.9250 sec 2024-12-08T00:16:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T00:16:48,261 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:48,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-08T00:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T00:16:48,264 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:48,264 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:48,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:48,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T00:16:48,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:48,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:48,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:48,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:48,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:48,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:48,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208875f0a59006f43d292d9d8884a56ce56_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617007669/Put/seqid=0 2024-12-08T00:16:48,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742361_1537 (size=12304) 2024-12-08T00:16:48,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:48,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208875f0a59006f43d292d9d8884a56ce56_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208875f0a59006f43d292d9d8884a56ce56_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:48,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/80ebe9a29e3a49318418fc61063bf9fe, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:48,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/80ebe9a29e3a49318418fc61063bf9fe is 175, key is test_row_0/A:col10/1733617007669/Put/seqid=0 2024-12-08T00:16:48,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742362_1538 (size=31105) 2024-12-08T00:16:48,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617068340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617068342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617068344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617068360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,362 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T00:16:48,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617068371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,372 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T00:16:48,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:48,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:48,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:48,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617068450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617068450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617068451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T00:16:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:48,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T00:16:48,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617068660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617068660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617068660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T00:16:48,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:48,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:48,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:48,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,739 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/80ebe9a29e3a49318418fc61063bf9fe 2024-12-08T00:16:48,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/2cf208c15e204325a46b3e3baba54bc3 is 50, key is test_row_0/B:col10/1733617007669/Put/seqid=0 2024-12-08T00:16:48,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742363_1539 (size=12151) 2024-12-08T00:16:48,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T00:16:48,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:48,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:48,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:48,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T00:16:48,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617068971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617068971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617068971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:49,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T00:16:49,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:49,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/2cf208c15e204325a46b3e3baba54bc3 2024-12-08T00:16:49,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/aa864c38cc9c4448a1812e3adace63f9 is 50, key is test_row_0/C:col10/1733617007669/Put/seqid=0 2024-12-08T00:16:49,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742364_1540 (size=12151) 2024-12-08T00:16:49,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T00:16:49,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:49,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T00:16:49,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T00:16:49,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617069481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617069481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617069481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:49,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T00:16:49,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:49,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:49,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/aa864c38cc9c4448a1812e3adace63f9 2024-12-08T00:16:49,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/80ebe9a29e3a49318418fc61063bf9fe as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/80ebe9a29e3a49318418fc61063bf9fe 2024-12-08T00:16:49,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/80ebe9a29e3a49318418fc61063bf9fe, entries=150, sequenceid=150, filesize=30.4 K 2024-12-08T00:16:49,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/2cf208c15e204325a46b3e3baba54bc3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/2cf208c15e204325a46b3e3baba54bc3 2024-12-08T00:16:49,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/2cf208c15e204325a46b3e3baba54bc3, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T00:16:49,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/aa864c38cc9c4448a1812e3adace63f9 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/aa864c38cc9c4448a1812e3adace63f9 2024-12-08T00:16:49,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/aa864c38cc9c4448a1812e3adace63f9, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T00:16:49,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 951f2f04d16f20e5ba0261e33c9a4276 in 1253ms, sequenceid=150, compaction requested=true 2024-12-08T00:16:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:49,568 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:49,568 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:49,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:49,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/B is initiating minor compaction (all files) 2024-12-08T00:16:49,569 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/B in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,569 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/5b10977367094da6be81b3c2f22acde1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/49a55d319cda4396a48c88f7e822526d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/2cf208c15e204325a46b3e3baba54bc3] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=35.6 K 2024-12-08T00:16:49,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:49,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/A is initiating minor compaction (all files) 2024-12-08T00:16:49,569 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/A in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b10977367094da6be81b3c2f22acde1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733617004165 2024-12-08T00:16:49,569 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/913a23e7f329403e8f47d707cace20ff, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2e719d416e0e4a29b45f8c7b69549942, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/80ebe9a29e3a49318418fc61063bf9fe] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=91.1 K 2024-12-08T00:16:49,569 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/913a23e7f329403e8f47d707cace20ff, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2e719d416e0e4a29b45f8c7b69549942, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/80ebe9a29e3a49318418fc61063bf9fe] 2024-12-08T00:16:49,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a55d319cda4396a48c88f7e822526d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733617005495 2024-12-08T00:16:49,569 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 913a23e7f329403e8f47d707cace20ff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733617004165 2024-12-08T00:16:49,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf208c15e204325a46b3e3baba54bc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733617007669 2024-12-08T00:16:49,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e719d416e0e4a29b45f8c7b69549942, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733617005495 2024-12-08T00:16:49,570 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ebe9a29e3a49318418fc61063bf9fe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733617007669 2024-12-08T00:16:49,575 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#B#compaction#453 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:49,575 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/f15795d240d44ea996d844ffaff759ab is 50, key is test_row_0/B:col10/1733617007669/Put/seqid=0 2024-12-08T00:16:49,576 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:49,577 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412087b7cbbfe97cb45fdb9f66dece4036de2_951f2f04d16f20e5ba0261e33c9a4276 store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:49,579 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412087b7cbbfe97cb45fdb9f66dece4036de2_951f2f04d16f20e5ba0261e33c9a4276, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:49,579 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087b7cbbfe97cb45fdb9f66dece4036de2_951f2f04d16f20e5ba0261e33c9a4276 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:49,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742365_1541 (size=12459) 2024-12-08T00:16:49,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742366_1542 (size=4469) 2024-12-08T00:16:49,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T00:16:49,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,638 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T00:16:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:49,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120855c5a9ed41914c3295a54add3292d837_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617008343/Put/seqid=0 2024-12-08T00:16:49,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742367_1543 (size=12304) 2024-12-08T00:16:49,983 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#A#compaction#454 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:49,984 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/f15795d240d44ea996d844ffaff759ab as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/f15795d240d44ea996d844ffaff759ab 2024-12-08T00:16:49,984 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/6651176560cd4f42a645537385bf73f7 is 175, key is test_row_0/A:col10/1733617007669/Put/seqid=0 2024-12-08T00:16:49,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742368_1544 (size=31413) 2024-12-08T00:16:49,989 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/B of 951f2f04d16f20e5ba0261e33c9a4276 into f15795d240d44ea996d844ffaff759ab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:49,989 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:49,989 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/B, priority=13, startTime=1733617009568; duration=0sec 2024-12-08T00:16:49,989 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:49,989 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:B 2024-12-08T00:16:49,989 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:49,989 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:49,990 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/C is initiating minor compaction (all files) 2024-12-08T00:16:49,990 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/C in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:49,990 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/00add05daa65473c93dfca57c4f29444, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/c7c65409aa4e4804b8379a019260c96d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/aa864c38cc9c4448a1812e3adace63f9] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=35.6 K 2024-12-08T00:16:49,990 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 00add05daa65473c93dfca57c4f29444, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733617004165 2024-12-08T00:16:49,990 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting c7c65409aa4e4804b8379a019260c96d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733617005495 2024-12-08T00:16:49,990 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting aa864c38cc9c4448a1812e3adace63f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733617007669 2024-12-08T00:16:49,991 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/6651176560cd4f42a645537385bf73f7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/6651176560cd4f42a645537385bf73f7 2024-12-08T00:16:49,995 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/A of 951f2f04d16f20e5ba0261e33c9a4276 into 6651176560cd4f42a645537385bf73f7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:49,995 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:49,995 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/A, priority=13, startTime=1733617009568; duration=0sec 2024-12-08T00:16:49,995 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:49,995 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:A 2024-12-08T00:16:49,996 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:49,996 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/1b85e17884ee4be085ed602484b8c593 is 50, key is test_row_0/C:col10/1733617007669/Put/seqid=0 2024-12-08T00:16:49,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742369_1545 (size=12459) 2024-12-08T00:16:50,002 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/1b85e17884ee4be085ed602484b8c593 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/1b85e17884ee4be085ed602484b8c593 2024-12-08T00:16:50,006 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/C of 951f2f04d16f20e5ba0261e33c9a4276 into 1b85e17884ee4be085ed602484b8c593(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:50,006 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:50,006 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/C, priority=13, startTime=1733617009568; duration=0sec 2024-12-08T00:16:50,006 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:50,006 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:C 2024-12-08T00:16:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,049 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120855c5a9ed41914c3295a54add3292d837_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120855c5a9ed41914c3295a54add3292d837_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:50,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/707491653f864c0f8ece2d6032c0cf53, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:50,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/707491653f864c0f8ece2d6032c0cf53 is 175, key is test_row_0/A:col10/1733617008343/Put/seqid=0 2024-12-08T00:16:50,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742370_1546 (size=31105) 2024-12-08T00:16:50,053 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/707491653f864c0f8ece2d6032c0cf53 2024-12-08T00:16:50,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/3722f3bb5c2b48d59232763614fdb560 is 50, key is test_row_0/B:col10/1733617008343/Put/seqid=0 2024-12-08T00:16:50,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742371_1547 (size=12151) 2024-12-08T00:16:50,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T00:16:50,463 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/3722f3bb5c2b48d59232763614fdb560 2024-12-08T00:16:50,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/6257d39706824bfd849f73fac1409ddb is 50, key is test_row_0/C:col10/1733617008343/Put/seqid=0 2024-12-08T00:16:50,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742372_1548 (size=12151) 2024-12-08T00:16:50,472 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/6257d39706824bfd849f73fac1409ddb 2024-12-08T00:16:50,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/707491653f864c0f8ece2d6032c0cf53 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/707491653f864c0f8ece2d6032c0cf53 2024-12-08T00:16:50,478 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/707491653f864c0f8ece2d6032c0cf53, entries=150, sequenceid=166, filesize=30.4 K 2024-12-08T00:16:50,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/3722f3bb5c2b48d59232763614fdb560 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/3722f3bb5c2b48d59232763614fdb560 2024-12-08T00:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,482 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/3722f3bb5c2b48d59232763614fdb560, entries=150, sequenceid=166, filesize=11.9 K 2024-12-08T00:16:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/6257d39706824bfd849f73fac1409ddb as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/6257d39706824bfd849f73fac1409ddb 2024-12-08T00:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/6257d39706824bfd849f73fac1409ddb, entries=150, sequenceid=166, filesize=11.9 K 2024-12-08T00:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 951f2f04d16f20e5ba0261e33c9a4276 in 848ms, sequenceid=166, compaction requested=false 2024-12-08T00:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-08T00:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-08T00:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-08T00:16:50,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2240 sec 2024-12-08T00:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.2280 sec 2024-12-08T00:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:50,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:16:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:50,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208270cdf74999a452f928bdd9d86fca052_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617010530/Put/seqid=0 2024-12-08T00:16:50,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742373_1549 (size=19774) 2024-12-08T00:16:50,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733617010550, max timestamp=1733617010559Num uniques 859; 2024-12-08T00:16:50,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:50,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733617010550, max timestamp=1733617010559Num uniques 859; 2024-12-08T00:16:50,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:50,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733617010550, max timestamp=1733617010559Num uniques 859; 2024-12-08T00:16:50,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:50,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617070581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:50,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:50,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617070585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617070586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:50,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617070690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:50,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617070701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:50,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617070701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617070900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617070910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617070911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:50,948 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:50,950 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208270cdf74999a452f928bdd9d86fca052_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208270cdf74999a452f928bdd9d86fca052_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:50,951 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/e72b485e4d274bd99d9a558611ef8bff, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:50,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/e72b485e4d274bd99d9a558611ef8bff is 175, key is test_row_0/A:col10/1733617010530/Put/seqid=0 2024-12-08T00:16:50,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742374_1550 (size=57033) 2024-12-08T00:16:51,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617071211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:51,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:51,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617071221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617071221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:51,354 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/e72b485e4d274bd99d9a558611ef8bff 2024-12-08T00:16:51,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/024ac240564443b987542ef6b545c269 is 50, key is test_row_0/B:col10/1733617010530/Put/seqid=0 2024-12-08T00:16:51,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742375_1551 (size=12151) 2024-12-08T00:16:51,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617071721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:51,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:51,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617071731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:51,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617071731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:51,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/024ac240564443b987542ef6b545c269 2024-12-08T00:16:51,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/61dbb714fb9d487a812d6076c790a1d6 is 50, key is test_row_0/C:col10/1733617010530/Put/seqid=0 2024-12-08T00:16:51,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742376_1552 (size=12151) 2024-12-08T00:16:52,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/61dbb714fb9d487a812d6076c790a1d6 2024-12-08T00:16:52,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/e72b485e4d274bd99d9a558611ef8bff as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/e72b485e4d274bd99d9a558611ef8bff 2024-12-08T00:16:52,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/e72b485e4d274bd99d9a558611ef8bff, entries=300, sequenceid=179, filesize=55.7 K 2024-12-08T00:16:52,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/024ac240564443b987542ef6b545c269 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/024ac240564443b987542ef6b545c269 2024-12-08T00:16:52,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/024ac240564443b987542ef6b545c269, entries=150, sequenceid=179, filesize=11.9 K 2024-12-08T00:16:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/61dbb714fb9d487a812d6076c790a1d6 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/61dbb714fb9d487a812d6076c790a1d6 2024-12-08T00:16:52,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/61dbb714fb9d487a812d6076c790a1d6, entries=150, sequenceid=179, filesize=11.9 K 2024-12-08T00:16:52,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 951f2f04d16f20e5ba0261e33c9a4276 in 1653ms, sequenceid=179, compaction requested=true 2024-12-08T00:16:52,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:52,185 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:52,185 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:52,185 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:52,185 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/A is initiating minor compaction (all files) 2024-12-08T00:16:52,186 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/A in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,186 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/6651176560cd4f42a645537385bf73f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/707491653f864c0f8ece2d6032c0cf53, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/e72b485e4d274bd99d9a558611ef8bff] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=116.7 K 2024-12-08T00:16:52,186 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,186 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/6651176560cd4f42a645537385bf73f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/707491653f864c0f8ece2d6032c0cf53, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/e72b485e4d274bd99d9a558611ef8bff] 2024-12-08T00:16:52,186 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6651176560cd4f42a645537385bf73f7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733617007669 2024-12-08T00:16:52,186 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 707491653f864c0f8ece2d6032c0cf53, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733617008339 2024-12-08T00:16:52,187 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting e72b485e4d274bd99d9a558611ef8bff, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733617010511 2024-12-08T00:16:52,187 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:52,187 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/B is initiating minor compaction (all files) 2024-12-08T00:16:52,187 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/B in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,188 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/f15795d240d44ea996d844ffaff759ab, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/3722f3bb5c2b48d59232763614fdb560, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/024ac240564443b987542ef6b545c269] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=35.9 K 2024-12-08T00:16:52,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting f15795d240d44ea996d844ffaff759ab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733617007669 2024-12-08T00:16:52,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 3722f3bb5c2b48d59232763614fdb560, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733617008339 2024-12-08T00:16:52,188 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 024ac240564443b987542ef6b545c269, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733617010527 2024-12-08T00:16:52,191 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:52,193 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412082c792b053c6c466eb4c659ec69802937_951f2f04d16f20e5ba0261e33c9a4276 store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:52,193 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:52,194 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/28785985a1c44bed9d9f8d3af1b2379d is 50, key is test_row_0/B:col10/1733617010530/Put/seqid=0 2024-12-08T00:16:52,195 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412082c792b053c6c466eb4c659ec69802937_951f2f04d16f20e5ba0261e33c9a4276, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:52,195 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082c792b053c6c466eb4c659ec69802937_951f2f04d16f20e5ba0261e33c9a4276 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:52,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742377_1553 (size=12561) 2024-12-08T00:16:52,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742378_1554 (size=4469) 2024-12-08T00:16:52,199 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#A#compaction#462 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:52,199 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/74a140fd6e714679a21266b497500f92 is 175, key is test_row_0/A:col10/1733617010530/Put/seqid=0 2024-12-08T00:16:52,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742379_1555 (size=31515) 2024-12-08T00:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:52,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-08T00:16:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:52,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120861ec48dfc4c34408a3375d4e879e4cf4_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617012393/Put/seqid=0 2024-12-08T00:16:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617072412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:52,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742380_1556 (size=14794) 2024-12-08T00:16:52,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617072414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T00:16:52,441 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:52,441 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-08T00:16:52,443 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T00:16:52,443 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:52,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:52,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617072520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:52,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617072530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T00:16:52,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T00:16:52,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:52,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:52,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:52,600 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/28785985a1c44bed9d9f8d3af1b2379d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/28785985a1c44bed9d9f8d3af1b2379d 2024-12-08T00:16:52,604 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/B of 951f2f04d16f20e5ba0261e33c9a4276 into 28785985a1c44bed9d9f8d3af1b2379d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:52,604 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:52,604 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/B, priority=13, startTime=1733617012185; duration=0sec 2024-12-08T00:16:52,604 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:52,604 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:B 2024-12-08T00:16:52,604 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T00:16:52,605 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T00:16:52,605 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/C is initiating minor compaction (all files) 2024-12-08T00:16:52,605 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/C in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,605 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/1b85e17884ee4be085ed602484b8c593, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/6257d39706824bfd849f73fac1409ddb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/61dbb714fb9d487a812d6076c790a1d6] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=35.9 K 2024-12-08T00:16:52,605 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b85e17884ee4be085ed602484b8c593, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733617007669 2024-12-08T00:16:52,605 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/74a140fd6e714679a21266b497500f92 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/74a140fd6e714679a21266b497500f92 2024-12-08T00:16:52,606 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 6257d39706824bfd849f73fac1409ddb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733617008339 2024-12-08T00:16:52,606 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 61dbb714fb9d487a812d6076c790a1d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733617010527 2024-12-08T00:16:52,610 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/A of 951f2f04d16f20e5ba0261e33c9a4276 into 74a140fd6e714679a21266b497500f92(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:52,610 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:52,610 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/A, priority=13, startTime=1733617012184; duration=0sec 2024-12-08T00:16:52,610 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:52,610 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:A 2024-12-08T00:16:52,611 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:52,611 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/396ca0f236dd42f681d913b18e36e154 is 50, key is test_row_0/C:col10/1733617010530/Put/seqid=0 2024-12-08T00:16:52,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742381_1557 (size=12561) 2024-12-08T00:16:52,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617072731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:52,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617072740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:52,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617072740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:52,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617072741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:52,742 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:52,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,742 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617072741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:52,743 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:52,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T00:16:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T00:16:52,816 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:52,819 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120861ec48dfc4c34408a3375d4e879e4cf4_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120861ec48dfc4c34408a3375d4e879e4cf4_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:52,819 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/0a4e327f478f45d7bcf7db782ac7ab2a, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:52,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/0a4e327f478f45d7bcf7db782ac7ab2a is 175, key is test_row_0/A:col10/1733617012393/Put/seqid=0 2024-12-08T00:16:52,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742382_1558 (size=39749) 2024-12-08T00:16:52,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T00:16:52,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:52,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:52,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:52,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,019 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/396ca0f236dd42f681d913b18e36e154 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/396ca0f236dd42f681d913b18e36e154 2024-12-08T00:16:53,025 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/C of 951f2f04d16f20e5ba0261e33c9a4276 into 396ca0f236dd42f681d913b18e36e154(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:53,025 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:53,025 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/C, priority=13, startTime=1733617012185; duration=0sec 2024-12-08T00:16:53,025 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:53,025 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:C 2024-12-08T00:16:53,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617073041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:53,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617073050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:53,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T00:16:53,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:53,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:53,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:53,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T00:16:53,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T00:16:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:53,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,224 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/0a4e327f478f45d7bcf7db782ac7ab2a 2024-12-08T00:16:53,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/109fb6ea0adb4fe08ef08922e5937e6f is 50, key is test_row_0/B:col10/1733617012393/Put/seqid=0 2024-12-08T00:16:53,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742383_1559 (size=12151) 2024-12-08T00:16:53,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/109fb6ea0adb4fe08ef08922e5937e6f 2024-12-08T00:16:53,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/a933b64ad4254064b104fba7e5205d87 is 50, key is test_row_0/C:col10/1733617012393/Put/seqid=0 2024-12-08T00:16:53,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742384_1560 (size=12151) 2024-12-08T00:16:53,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T00:16:53,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:53,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:53,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:53,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T00:16:53,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:53,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:53,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:53,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:53,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617073551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:53,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617073560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T00:16:53,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/a933b64ad4254064b104fba7e5205d87 2024-12-08T00:16:53,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/0a4e327f478f45d7bcf7db782ac7ab2a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/0a4e327f478f45d7bcf7db782ac7ab2a 2024-12-08T00:16:53,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/0a4e327f478f45d7bcf7db782ac7ab2a, entries=200, sequenceid=216, filesize=38.8 K 2024-12-08T00:16:53,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/109fb6ea0adb4fe08ef08922e5937e6f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/109fb6ea0adb4fe08ef08922e5937e6f 2024-12-08T00:16:53,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/109fb6ea0adb4fe08ef08922e5937e6f, entries=150, sequenceid=216, filesize=11.9 K 2024-12-08T00:16:53,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/a933b64ad4254064b104fba7e5205d87 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a933b64ad4254064b104fba7e5205d87 2024-12-08T00:16:53,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a933b64ad4254064b104fba7e5205d87, entries=150, sequenceid=216, filesize=11.9 K 2024-12-08T00:16:53,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 951f2f04d16f20e5ba0261e33c9a4276 in 1261ms, sequenceid=216, compaction requested=false 2024-12-08T00:16:53,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:53,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T00:16:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:53,665 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-08T00:16:53,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:53,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:53,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:53,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:53,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:53,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:53,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d6345fbc4b62405f9138e95a4f924e88_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617012411/Put/seqid=0 2024-12-08T00:16:53,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742385_1561 (size=12304) 2024-12-08T00:16:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:54,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d6345fbc4b62405f9138e95a4f924e88_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d6345fbc4b62405f9138e95a4f924e88_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/024daa4a07a04989b87330fda6e3d078, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/024daa4a07a04989b87330fda6e3d078 is 175, key is test_row_0/A:col10/1733617012411/Put/seqid=0 2024-12-08T00:16:54,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742386_1562 (size=31105) 2024-12-08T00:16:54,089 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/024daa4a07a04989b87330fda6e3d078 2024-12-08T00:16:54,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/797be330beda4d52a03d3cbe05c96a29 is 50, key is test_row_0/B:col10/1733617012411/Put/seqid=0 2024-12-08T00:16:54,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742387_1563 (size=12151) 2024-12-08T00:16:54,519 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/797be330beda4d52a03d3cbe05c96a29 2024-12-08T00:16:54,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/b54d76607a6f400f9f38bc4a52168795 is 50, key is test_row_0/C:col10/1733617012411/Put/seqid=0 2024-12-08T00:16:54,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742388_1564 (size=12151) 2024-12-08T00:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:54,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T00:16:54,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733617014595, max timestamp=1733617014606Num uniques 859; 2024-12-08T00:16:54,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:54,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733617014595, max timestamp=1733617014606Num uniques 859; 2024-12-08T00:16:54,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:54,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733617014595, max timestamp=1733617014606Num uniques 859; 2024-12-08T00:16:54,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:54,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617074643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:54,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617074644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617074751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617074751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:54,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617074760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:54,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617074761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:54,764 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:54,764 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:54,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617074771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:54,775 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:54,928 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/b54d76607a6f400f9f38bc4a52168795 2024-12-08T00:16:54,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/024daa4a07a04989b87330fda6e3d078 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/024daa4a07a04989b87330fda6e3d078 2024-12-08T00:16:54,938 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/024daa4a07a04989b87330fda6e3d078, entries=150, sequenceid=228, filesize=30.4 K 2024-12-08T00:16:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/797be330beda4d52a03d3cbe05c96a29 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/797be330beda4d52a03d3cbe05c96a29 2024-12-08T00:16:54,946 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/797be330beda4d52a03d3cbe05c96a29, entries=150, sequenceid=228, filesize=11.9 K 2024-12-08T00:16:54,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/b54d76607a6f400f9f38bc4a52168795 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/b54d76607a6f400f9f38bc4a52168795 2024-12-08T00:16:54,952 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/b54d76607a6f400f9f38bc4a52168795, entries=150, sequenceid=228, filesize=11.9 K 2024-12-08T00:16:54,952 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 951f2f04d16f20e5ba0261e33c9a4276 in 1287ms, sequenceid=228, compaction requested=true 2024-12-08T00:16:54,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:54,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:54,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-08T00:16:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-08T00:16:54,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-08T00:16:54,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5100 sec 2024-12-08T00:16:54,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.5150 sec 2024-12-08T00:16:54,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-08T00:16:54,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:54,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T00:16:54,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:54,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T00:16:54,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:54,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T00:16:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:54,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120887bcf32b8cb0488e942d913e47c807a6_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617014962/Put/seqid=0 2024-12-08T00:16:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617074978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:54,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617074979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:55,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742389_1565 (size=14994) 2024-12-08T00:16:55,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617075091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617075091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:55,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617075301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617075301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:55,402 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,405 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120887bcf32b8cb0488e942d913e47c807a6_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120887bcf32b8cb0488e942d913e47c807a6_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:55,406 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/5dac596401604a9ba5156c57f136754b, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:55,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/5dac596401604a9ba5156c57f136754b is 175, key is test_row_0/A:col10/1733617014962/Put/seqid=0 2024-12-08T00:16:55,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742390_1566 (size=39949) 2024-12-08T00:16:55,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/5dac596401604a9ba5156c57f136754b 2024-12-08T00:16:55,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8237ee88cbb348c1894a15148f73eb64 is 50, key is test_row_0/B:col10/1733617014962/Put/seqid=0 2024-12-08T00:16:55,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742391_1567 (size=12301) 2024-12-08T00:16:55,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617075611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:55,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617075611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:55,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8237ee88cbb348c1894a15148f73eb64 2024-12-08T00:16:55,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/e9632f29c8664f8f8642827b4a866b0a is 50, key is test_row_0/C:col10/1733617014962/Put/seqid=0 2024-12-08T00:16:55,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742392_1568 (size=12301) 2024-12-08T00:16:55,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/e9632f29c8664f8f8642827b4a866b0a 2024-12-08T00:16:55,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/5dac596401604a9ba5156c57f136754b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5dac596401604a9ba5156c57f136754b 2024-12-08T00:16:55,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5dac596401604a9ba5156c57f136754b, entries=200, sequenceid=266, filesize=39.0 K 2024-12-08T00:16:55,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8237ee88cbb348c1894a15148f73eb64 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8237ee88cbb348c1894a15148f73eb64 2024-12-08T00:16:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8237ee88cbb348c1894a15148f73eb64, entries=150, sequenceid=266, filesize=12.0 K 2024-12-08T00:16:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/e9632f29c8664f8f8642827b4a866b0a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e9632f29c8664f8f8642827b4a866b0a 2024-12-08T00:16:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e9632f29c8664f8f8642827b4a866b0a, entries=150, sequenceid=266, filesize=12.0 K 2024-12-08T00:16:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 951f2f04d16f20e5ba0261e33c9a4276 in 921ms, sequenceid=266, compaction requested=true 2024-12-08T00:16:55,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:55,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:55,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T00:16:55,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:55,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-08T00:16:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,884 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:55,885 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,887 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49164 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:55,887 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/C is initiating minor compaction (all files) 2024-12-08T00:16:55,887 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/C in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,887 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142318 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:55,887 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/396ca0f236dd42f681d913b18e36e154, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a933b64ad4254064b104fba7e5205d87, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/b54d76607a6f400f9f38bc4a52168795, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e9632f29c8664f8f8642827b4a866b0a] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=48.0 K 2024-12-08T00:16:55,887 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/A is initiating minor compaction (all files) 2024-12-08T00:16:55,887 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/A in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:55,887 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/74a140fd6e714679a21266b497500f92, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/0a4e327f478f45d7bcf7db782ac7ab2a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/024daa4a07a04989b87330fda6e3d078, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5dac596401604a9ba5156c57f136754b] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=139.0 K 2024-12-08T00:16:55,887 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:55,887 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/74a140fd6e714679a21266b497500f92, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/0a4e327f478f45d7bcf7db782ac7ab2a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/024daa4a07a04989b87330fda6e3d078, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5dac596401604a9ba5156c57f136754b] 2024-12-08T00:16:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,887 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 396ca0f236dd42f681d913b18e36e154, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733617010527 2024-12-08T00:16:55,887 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a140fd6e714679a21266b497500f92, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733617010527 2024-12-08T00:16:55,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a933b64ad4254064b104fba7e5205d87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733617010578 2024-12-08T00:16:55,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a4e327f478f45d7bcf7db782ac7ab2a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733617010578 2024-12-08T00:16:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting b54d76607a6f400f9f38bc4a52168795, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733617012409 2024-12-08T00:16:55,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024daa4a07a04989b87330fda6e3d078, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733617012409 2024-12-08T00:16:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting e9632f29c8664f8f8642827b4a866b0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733617014638 2024-12-08T00:16:55,888 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dac596401604a9ba5156c57f136754b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733617014638 2024-12-08T00:16:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,898 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,899 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,899 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/fbb6683ab89542889e79a1e8595f2761 is 50, key is test_row_0/C:col10/1733617014962/Put/seqid=0 2024-12-08T00:16:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,911 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412087835059aa1ee4fafb0ddb484a98dad2e_951f2f04d16f20e5ba0261e33c9a4276 store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,913 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412087835059aa1ee4fafb0ddb484a98dad2e_951f2f04d16f20e5ba0261e33c9a4276, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:55,913 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087835059aa1ee4fafb0ddb484a98dad2e_951f2f04d16f20e5ba0261e33c9a4276 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742393_1569 (size=12847) 2024-12-08T00:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,922 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/fbb6683ab89542889e79a1e8595f2761 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/fbb6683ab89542889e79a1e8595f2761 2024-12-08T00:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,925 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/C of 951f2f04d16f20e5ba0261e33c9a4276 into fbb6683ab89542889e79a1e8595f2761(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:55,925 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,925 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/C, priority=12, startTime=1733617015884; duration=0sec 2024-12-08T00:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,925 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:55,925 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:C 2024-12-08T00:16:55,925 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,926 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49164 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,926 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/B is initiating minor compaction (all files) 2024-12-08T00:16:55,926 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/B in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:55,926 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/28785985a1c44bed9d9f8d3af1b2379d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/109fb6ea0adb4fe08ef08922e5937e6f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/797be330beda4d52a03d3cbe05c96a29, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8237ee88cbb348c1894a15148f73eb64] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=48.0 K 2024-12-08T00:16:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,927 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 28785985a1c44bed9d9f8d3af1b2379d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733617010527 2024-12-08T00:16:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,927 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 109fb6ea0adb4fe08ef08922e5937e6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733617010578 2024-12-08T00:16:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,928 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 797be330beda4d52a03d3cbe05c96a29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733617012409 2024-12-08T00:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,929 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8237ee88cbb348c1894a15148f73eb64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733617014638 2024-12-08T00:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742394_1570 (size=4469) 2024-12-08T00:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,939 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#A#compaction#475 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,939 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/531c7ddc60f040abaced466e9f4ba79d is 175, key is test_row_0/A:col10/1733617014962/Put/seqid=0 2024-12-08T00:16:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,940 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#B#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:55,940 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8b88fe0dae5646b9b2c3bca8a6b6216b is 50, key is test_row_0/B:col10/1733617014962/Put/seqid=0 2024-12-08T00:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742395_1571 (size=31801) 2024-12-08T00:16:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742396_1572 (size=12847) 2024-12-08T00:16:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:56,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T00:16:56,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:56,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:56,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:56,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:56,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:56,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a0aba96ae3cd4cce88983f2db84ddbf1_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617014977/Put/seqid=0 2024-12-08T00:16:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742397_1573 (size=14994) 2024-12-08T00:16:56,147 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,150 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a0aba96ae3cd4cce88983f2db84ddbf1_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a0aba96ae3cd4cce88983f2db84ddbf1_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:56,151 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/5b30981ed74444e8980b2a3360caa567, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:56,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/5b30981ed74444e8980b2a3360caa567 is 175, key is test_row_0/A:col10/1733617014977/Put/seqid=0 2024-12-08T00:16:56,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742398_1574 (size=39949) 2024-12-08T00:16:56,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733617016174, max timestamp=1733617016183Num uniques 859; 2024-12-08T00:16:56,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733617016174, max timestamp=1733617016183Num uniques 859; 2024-12-08T00:16:56,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:56,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:56,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733617016174, max timestamp=1733617016183Num uniques 859; 2024-12-08T00:16:56,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c1a8cf8d4cfa:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T00:16:56,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617076219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:56,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617076225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:56,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617076331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:56,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617076340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:56,349 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/531c7ddc60f040abaced466e9f4ba79d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/531c7ddc60f040abaced466e9f4ba79d 2024-12-08T00:16:56,356 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/A of 951f2f04d16f20e5ba0261e33c9a4276 into 531c7ddc60f040abaced466e9f4ba79d(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:56,356 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:56,356 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/A, priority=12, startTime=1733617015884; duration=0sec 2024-12-08T00:16:56,356 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:56,356 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:A 2024-12-08T00:16:56,358 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8b88fe0dae5646b9b2c3bca8a6b6216b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8b88fe0dae5646b9b2c3bca8a6b6216b 2024-12-08T00:16:56,362 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/B of 951f2f04d16f20e5ba0261e33c9a4276 into 8b88fe0dae5646b9b2c3bca8a6b6216b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:56,362 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:56,362 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/B, priority=12, startTime=1733617015884; duration=0sec 2024-12-08T00:16:56,362 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:56,362 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:B 2024-12-08T00:16:56,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617076541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:56,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617076550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:56,562 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/5b30981ed74444e8980b2a3360caa567 2024-12-08T00:16:56,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/a9c09d68a1ce48e18be0203e79ece9c4 is 50, key is test_row_0/B:col10/1733617014977/Put/seqid=0 2024-12-08T00:16:56,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742399_1575 (size=12301) 2024-12-08T00:16:56,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/a9c09d68a1ce48e18be0203e79ece9c4 2024-12-08T00:16:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T00:16:56,611 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:56,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-08T00:16:56,613 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T00:16:56,614 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:56,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:56,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/25346b1818c542499e8685c80635a24e is 50, key is test_row_0/C:col10/1733617014977/Put/seqid=0 2024-12-08T00:16:56,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742400_1576 (size=12301) 2024-12-08T00:16:56,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/25346b1818c542499e8685c80635a24e 2024-12-08T00:16:56,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/5b30981ed74444e8980b2a3360caa567 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5b30981ed74444e8980b2a3360caa567 2024-12-08T00:16:56,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5b30981ed74444e8980b2a3360caa567, entries=200, sequenceid=277, filesize=39.0 K 2024-12-08T00:16:56,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/a9c09d68a1ce48e18be0203e79ece9c4 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a9c09d68a1ce48e18be0203e79ece9c4 2024-12-08T00:16:56,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a9c09d68a1ce48e18be0203e79ece9c4, entries=150, sequenceid=277, filesize=12.0 K 2024-12-08T00:16:56,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/25346b1818c542499e8685c80635a24e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/25346b1818c542499e8685c80635a24e 2024-12-08T00:16:56,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/25346b1818c542499e8685c80635a24e, entries=150, sequenceid=277, filesize=12.0 K 2024-12-08T00:16:56,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 951f2f04d16f20e5ba0261e33c9a4276 in 548ms, sequenceid=277, compaction requested=false 2024-12-08T00:16:56,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T00:16:56,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T00:16:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:56,766 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T00:16:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T00:16:56,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208960044b3abda4e34a255b7819397ff9d_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617016202/Put/seqid=0 2024-12-08T00:16:56,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742401_1577 (size=12454) 2024-12-08T00:16:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:56,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208960044b3abda4e34a255b7819397ff9d_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208960044b3abda4e34a255b7819397ff9d_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:56,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/38b3766c657243b0848b14b9e48f262b, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:56,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/38b3766c657243b0848b14b9e48f262b is 175, key is test_row_0/A:col10/1733617016202/Put/seqid=0 2024-12-08T00:16:56,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742402_1578 (size=31255) 2024-12-08T00:16:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:56,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:56,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617076865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:56,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617076866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T00:16:56,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617076971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617076971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:57,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617077181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:57,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617077181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:57,193 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/38b3766c657243b0848b14b9e48f262b 2024-12-08T00:16:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/d1c851a7fa084abcb229d1a238acb6c3 is 50, key is test_row_0/B:col10/1733617016202/Put/seqid=0 2024-12-08T00:16:57,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742403_1579 (size=12301) 2024-12-08T00:16:57,237 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/d1c851a7fa084abcb229d1a238acb6c3 2024-12-08T00:16:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T00:16:57,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/a8d048e1e6104130ac45751d6ccdf867 is 50, key is test_row_0/C:col10/1733617016202/Put/seqid=0 2024-12-08T00:16:57,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742404_1580 (size=12301) 2024-12-08T00:16:57,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617077491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:57,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617077491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:57,707 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/a8d048e1e6104130ac45751d6ccdf867 2024-12-08T00:16:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/38b3766c657243b0848b14b9e48f262b as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/38b3766c657243b0848b14b9e48f262b 2024-12-08T00:16:57,719 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/38b3766c657243b0848b14b9e48f262b, entries=150, sequenceid=315, filesize=30.5 K 2024-12-08T00:16:57,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/d1c851a7fa084abcb229d1a238acb6c3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/d1c851a7fa084abcb229d1a238acb6c3 2024-12-08T00:16:57,730 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/d1c851a7fa084abcb229d1a238acb6c3, entries=150, sequenceid=315, filesize=12.0 K 2024-12-08T00:16:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/a8d048e1e6104130ac45751d6ccdf867 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a8d048e1e6104130ac45751d6ccdf867 2024-12-08T00:16:57,747 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a8d048e1e6104130ac45751d6ccdf867, entries=150, sequenceid=315, filesize=12.0 K 2024-12-08T00:16:57,751 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 951f2f04d16f20e5ba0261e33c9a4276 in 984ms, sequenceid=315, compaction requested=true 2024-12-08T00:16:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T00:16:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-08T00:16:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-08T00:16:57,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-08T00:16:57,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1390 sec 2024-12-08T00:16:57,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.1460 sec 2024-12-08T00:16:58,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:58,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T00:16:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:58,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080311898b7fba47ebb1c81a23687d2989_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617016859/Put/seqid=0 2024-12-08T00:16:58,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742405_1581 (size=14994) 2024-12-08T00:16:58,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617078040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617078043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617078150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617078151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617078361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617078361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,411 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:58,462 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080311898b7fba47ebb1c81a23687d2989_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080311898b7fba47ebb1c81a23687d2989_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:58,463 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/88cc8ffec1fc43d2884b934ea4c910cb, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:58,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/88cc8ffec1fc43d2884b934ea4c910cb is 175, key is test_row_0/A:col10/1733617016859/Put/seqid=0 2024-12-08T00:16:58,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742406_1582 (size=39949) 2024-12-08T00:16:58,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617078670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617078671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T00:16:58,771 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:16:58,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T00:16:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-08T00:16:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T00:16:58,774 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T00:16:58,774 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T00:16:58,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T00:16:58,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38072 deadline: 1733617078781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,783 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_2' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733617078802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,805 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_1' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38124 deadline: 1733617078811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:58,814 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c1a8cf8d4cfa,36021,1733616847970 for 'test_row_0' in 951f2f04d16f20e5ba0261e33c9a4276 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T00:16:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:58,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/88cc8ffec1fc43d2884b934ea4c910cb 2024-12-08T00:16:58,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/69bbafaffd684fbd81f918ad4747a16f is 50, key is test_row_0/B:col10/1733617016859/Put/seqid=0 2024-12-08T00:16:58,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742407_1583 (size=12301) 2024-12-08T00:16:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T00:16:58,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T00:16:58,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:58,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:58,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:58,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:58,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T00:16:59,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:59,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T00:16:59,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38112 deadline: 1733617079181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:59,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T00:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36021 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38026 deadline: 1733617079181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 , the old value is region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T00:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., hostname=c1a8cf8d4cfa,36021,1733616847970, seqNum=5 because the exception is null or not the one we care about 2024-12-08T00:16:59,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T00:16:59,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:59,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/69bbafaffd684fbd81f918ad4747a16f 2024-12-08T00:16:59,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/4a4bf4d1bba5417b80f2a812f506920d is 50, key is test_row_0/C:col10/1733617016859/Put/seqid=0 2024-12-08T00:16:59,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742408_1584 (size=12301) 2024-12-08T00:16:59,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T00:16:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T00:16:59,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T00:16:59,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:59,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/4a4bf4d1bba5417b80f2a812f506920d 2024-12-08T00:16:59,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T00:16:59,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:16:59,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T00:16:59,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/88cc8ffec1fc43d2884b934ea4c910cb as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/88cc8ffec1fc43d2884b934ea4c910cb 2024-12-08T00:16:59,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/88cc8ffec1fc43d2884b934ea4c910cb, entries=200, sequenceid=327, filesize=39.0 K 2024-12-08T00:16:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/69bbafaffd684fbd81f918ad4747a16f as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/69bbafaffd684fbd81f918ad4747a16f 2024-12-08T00:16:59,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/69bbafaffd684fbd81f918ad4747a16f, entries=150, sequenceid=327, filesize=12.0 K 2024-12-08T00:16:59,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/4a4bf4d1bba5417b80f2a812f506920d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/4a4bf4d1bba5417b80f2a812f506920d 2024-12-08T00:16:59,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/4a4bf4d1bba5417b80f2a812f506920d, entries=150, sequenceid=327, filesize=12.0 K 2024-12-08T00:16:59,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 951f2f04d16f20e5ba0261e33c9a4276 in 1717ms, sequenceid=327, compaction requested=true 2024-12-08T00:16:59,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:59,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T00:16:59,720 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:59,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:59,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T00:16:59,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:16:59,720 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:59,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951f2f04d16f20e5ba0261e33c9a4276:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T00:16:59,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:59,722 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:59,722 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:59,722 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/A is initiating minor compaction (all files) 2024-12-08T00:16:59,722 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/B is initiating minor compaction (all files) 2024-12-08T00:16:59,722 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/A in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,722 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/B in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,722 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8b88fe0dae5646b9b2c3bca8a6b6216b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a9c09d68a1ce48e18be0203e79ece9c4, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/d1c851a7fa084abcb229d1a238acb6c3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/69bbafaffd684fbd81f918ad4747a16f] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=48.6 K 2024-12-08T00:16:59,722 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/531c7ddc60f040abaced466e9f4ba79d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5b30981ed74444e8980b2a3360caa567, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/38b3766c657243b0848b14b9e48f262b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/88cc8ffec1fc43d2884b934ea4c910cb] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=139.6 K 2024-12-08T00:16:59,722 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,723 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. files: [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/531c7ddc60f040abaced466e9f4ba79d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5b30981ed74444e8980b2a3360caa567, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/38b3766c657243b0848b14b9e48f262b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/88cc8ffec1fc43d2884b934ea4c910cb] 2024-12-08T00:16:59,723 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b88fe0dae5646b9b2c3bca8a6b6216b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733617014638 2024-12-08T00:16:59,723 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 531c7ddc60f040abaced466e9f4ba79d, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733617014638 2024-12-08T00:16:59,723 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c09d68a1ce48e18be0203e79ece9c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733617014977 2024-12-08T00:16:59,724 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c851a7fa084abcb229d1a238acb6c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733617016202 2024-12-08T00:16:59,724 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b30981ed74444e8980b2a3360caa567, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733617014977 2024-12-08T00:16:59,724 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] compactions.Compactor(225): Compacting 69bbafaffd684fbd81f918ad4747a16f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733617016859 2024-12-08T00:16:59,724 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38b3766c657243b0848b14b9e48f262b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733617016202 2024-12-08T00:16:59,724 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88cc8ffec1fc43d2884b934ea4c910cb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733617016859 2024-12-08T00:16:59,734 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:59,735 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:59,735 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8fd3c2b98f0e4e5f8264a0e173554e7d is 50, key is test_row_0/B:col10/1733617016859/Put/seqid=0 2024-12-08T00:16:59,737 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120815087fe75402460db851e6ca99b21982_951f2f04d16f20e5ba0261e33c9a4276 store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:59,740 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120815087fe75402460db851e6ca99b21982_951f2f04d16f20e5ba0261e33c9a4276, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:59,740 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120815087fe75402460db851e6ca99b21982_951f2f04d16f20e5ba0261e33c9a4276 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:59,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742410_1586 (size=4469) 2024-12-08T00:16:59,752 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#A#compaction#487 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:59,752 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/af5895b99ece459e962710cf40fcda4d is 175, key is test_row_0/A:col10/1733617016859/Put/seqid=0 2024-12-08T00:16:59,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742411_1587 (size=31937) 2024-12-08T00:16:59,767 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/af5895b99ece459e962710cf40fcda4d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/af5895b99ece459e962710cf40fcda4d 2024-12-08T00:16:59,771 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/A of 951f2f04d16f20e5ba0261e33c9a4276 into af5895b99ece459e962710cf40fcda4d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:16:59,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742409_1585 (size=12983) 2024-12-08T00:16:59,771 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:16:59,771 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/A, priority=12, startTime=1733617019720; duration=0sec 2024-12-08T00:16:59,771 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T00:16:59,771 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:A 2024-12-08T00:16:59,771 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T00:16:59,772 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T00:16:59,772 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1541): 951f2f04d16f20e5ba0261e33c9a4276/C is initiating minor compaction (all files) 2024-12-08T00:16:59,772 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951f2f04d16f20e5ba0261e33c9a4276/C in TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,773 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/fbb6683ab89542889e79a1e8595f2761, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/25346b1818c542499e8685c80635a24e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a8d048e1e6104130ac45751d6ccdf867, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/4a4bf4d1bba5417b80f2a812f506920d] into tmpdir=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp, totalSize=48.6 K 2024-12-08T00:16:59,773 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb6683ab89542889e79a1e8595f2761, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733617014638 2024-12-08T00:16:59,774 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25346b1818c542499e8685c80635a24e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733617014977 2024-12-08T00:16:59,774 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d048e1e6104130ac45751d6ccdf867, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733617016202 2024-12-08T00:16:59,774 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4bf4d1bba5417b80f2a812f506920d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733617016859 2024-12-08T00:16:59,783 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951f2f04d16f20e5ba0261e33c9a4276#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T00:16:59,783 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/86361b15f1ea48338152ee483472d321 is 50, key is test_row_0/C:col10/1733617016859/Put/seqid=0 2024-12-08T00:16:59,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742412_1588 (size=12983) 2024-12-08T00:16:59,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36021 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T00:16:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:16:59,846 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T00:16:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:16:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:16:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:16:59,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:16:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088c7b2a03003c46909041384950d77aa8_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617018027/Put/seqid=0 2024-12-08T00:16:59,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742413_1589 (size=12454) 2024-12-08T00:16:59,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:16:59,860 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088c7b2a03003c46909041384950d77aa8_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088c7b2a03003c46909041384950d77aa8_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:16:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/f97543c7bbb7428690334bc7e6e8f799, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:16:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/f97543c7bbb7428690334bc7e6e8f799 is 175, key is test_row_0/A:col10/1733617018027/Put/seqid=0 2024-12-08T00:16:59,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742414_1590 (size=31255) 2024-12-08T00:16:59,864 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/f97543c7bbb7428690334bc7e6e8f799 2024-12-08T00:16:59,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/e9c03bcd002041019b90e4a7874f7f80 is 50, key is test_row_0/B:col10/1733617018027/Put/seqid=0 2024-12-08T00:16:59,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742415_1591 (size=12301) 2024-12-08T00:16:59,886 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/e9c03bcd002041019b90e4a7874f7f80 2024-12-08T00:16:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/30dc484fed2749e8a3d83b8a38ba4c45 is 50, key is test_row_0/C:col10/1733617018027/Put/seqid=0 2024-12-08T00:16:59,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742416_1592 (size=12301) 2024-12-08T00:16:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T00:16:59,930 INFO [Thread-2269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2269. 2024-12-08T00:16:59,930 DEBUG [Thread-2269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:59,930 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,930 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:59,932 INFO [Thread-2271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2271. 2024-12-08T00:16:59,932 DEBUG [Thread-2271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:59,932 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,932 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:59,933 INFO [Thread-2267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2267. 2024-12-08T00:16:59,933 DEBUG [Thread-2267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:59,933 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,933 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:59,934 INFO [Thread-2275 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2275. 2024-12-08T00:16:59,934 DEBUG [Thread-2275 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:59,934 DEBUG [Thread-2275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,934 DEBUG [Thread-2275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:16:59,937 INFO [Thread-2273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2273. 2024-12-08T00:16:59,937 DEBUG [Thread-2273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:16:59,937 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,937 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:16:59,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:17:00,180 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8fd3c2b98f0e4e5f8264a0e173554e7d as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8fd3c2b98f0e4e5f8264a0e173554e7d 2024-12-08T00:17:00,187 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/B of 951f2f04d16f20e5ba0261e33c9a4276 into 8fd3c2b98f0e4e5f8264a0e173554e7d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:17:00,187 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:17:00,187 INFO [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/B, priority=12, startTime=1733617019720; duration=0sec 2024-12-08T00:17:00,187 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:17:00,187 DEBUG [RS:0;c1a8cf8d4cfa:36021-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:B 2024-12-08T00:17:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36021 {}] regionserver.HRegion(8855): Flush requested on 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:00,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. as already flushing 2024-12-08T00:17:00,193 INFO [Thread-2258 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2258. 2024-12-08T00:17:00,193 INFO [Thread-2260 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2260. 2024-12-08T00:17:00,193 DEBUG [Thread-2258 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:17:00,193 DEBUG [Thread-2260 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:17:00,193 DEBUG [Thread-2258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:00,193 DEBUG [Thread-2260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:00,193 DEBUG [Thread-2258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:00,193 DEBUG [Thread-2260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:00,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:17:00,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:17:00,201 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/86361b15f1ea48338152ee483472d321 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/86361b15f1ea48338152ee483472d321 2024-12-08T00:17:00,205 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951f2f04d16f20e5ba0261e33c9a4276/C of 951f2f04d16f20e5ba0261e33c9a4276 into 86361b15f1ea48338152ee483472d321(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T00:17:00,205 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:17:00,205 INFO [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., storeName=951f2f04d16f20e5ba0261e33c9a4276/C, priority=12, startTime=1733617019720; duration=0sec 2024-12-08T00:17:00,205 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T00:17:00,205 DEBUG [RS:0;c1a8cf8d4cfa:36021-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951f2f04d16f20e5ba0261e33c9a4276:C 2024-12-08T00:17:00,301 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/30dc484fed2749e8a3d83b8a38ba4c45 2024-12-08T00:17:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/f97543c7bbb7428690334bc7e6e8f799 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/f97543c7bbb7428690334bc7e6e8f799 2024-12-08T00:17:00,315 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/f97543c7bbb7428690334bc7e6e8f799, entries=150, sequenceid=348, filesize=30.5 K 2024-12-08T00:17:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/e9c03bcd002041019b90e4a7874f7f80 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e9c03bcd002041019b90e4a7874f7f80 2024-12-08T00:17:00,319 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e9c03bcd002041019b90e4a7874f7f80, entries=150, sequenceid=348, filesize=12.0 K 2024-12-08T00:17:00,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/30dc484fed2749e8a3d83b8a38ba4c45 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/30dc484fed2749e8a3d83b8a38ba4c45 2024-12-08T00:17:00,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/30dc484fed2749e8a3d83b8a38ba4c45, entries=150, sequenceid=348, filesize=12.0 K 2024-12-08T00:17:00,324 INFO [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 951f2f04d16f20e5ba0261e33c9a4276 in 477ms, sequenceid=348, compaction requested=false 2024-12-08T00:17:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 951f2f04d16f20e5ba0261e33c9a4276: 2024-12-08T00:17:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:17:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c1a8cf8d4cfa:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-08T00:17:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-08T00:17:00,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-08T00:17:00,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5510 sec 2024-12-08T00:17:00,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.5550 sec 2024-12-08T00:17:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T00:17:00,922 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:17:06,244 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T00:17:08,875 INFO [Thread-2256 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2256. 2024-12-08T00:17:08,876 DEBUG [Thread-2256 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:17:08,876 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:08,876 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:08,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:17:08,884 INFO [Thread-2262 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2262. 2024-12-08T00:17:08,884 DEBUG [Thread-2262 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:17:08,884 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:08,884 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:08,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:17:08,903 INFO [Thread-2264 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2264. 2024-12-08T00:17:08,903 DEBUG [Thread-2264 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T00:17:08,903 DEBUG [Thread-2264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:08,903 DEBUG [Thread-2264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:08,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4204 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3981 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3951 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4195 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3988 2024-12-08T00:17:08,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T00:17:08,905 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T00:17:08,905 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T00:17:08,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:08,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:08,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:17:08,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T00:17:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T00:17:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T00:17:08,911 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733617028911"}]},"ts":"1733617028911"} 2024-12-08T00:17:08,913 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T00:17:08,913 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T00:17:08,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T00:17:08,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, UNASSIGN}] 2024-12-08T00:17:08,917 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, UNASSIGN 2024-12-08T00:17:08,918 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=951f2f04d16f20e5ba0261e33c9a4276, regionState=CLOSING, regionLocation=c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:17:08,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, UNASSIGN because future has completed 2024-12-08T00:17:08,921 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T00:17:08,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970}] 2024-12-08T00:17:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T00:17:09,075 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:09,075 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T00:17:09,075 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 951f2f04d16f20e5ba0261e33c9a4276, disabling compactions & flushes 2024-12-08T00:17:09,075 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:17:09,075 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:17:09,076 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. after waiting 0 ms 2024-12-08T00:17:09,076 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:17:09,076 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 951f2f04d16f20e5ba0261e33c9a4276 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T00:17:09,076 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=A 2024-12-08T00:17:09,077 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:17:09,077 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=B 2024-12-08T00:17:09,077 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:17:09,077 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951f2f04d16f20e5ba0261e33c9a4276, store=C 2024-12-08T00:17:09,077 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T00:17:09,085 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084ad2fb73b6e2431a8e28c8e611993013_951f2f04d16f20e5ba0261e33c9a4276 is 50, key is test_row_0/A:col10/1733617028882/Put/seqid=0 2024-12-08T00:17:09,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742417_1593 (size=12454) 2024-12-08T00:17:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T00:17:09,490 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T00:17:09,498 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084ad2fb73b6e2431a8e28c8e611993013_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084ad2fb73b6e2431a8e28c8e611993013_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:09,500 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/1ccc96541ab742d3b8a4e72c8e572262, store: [table=TestAcidGuarantees family=A region=951f2f04d16f20e5ba0261e33c9a4276] 2024-12-08T00:17:09,500 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/1ccc96541ab742d3b8a4e72c8e572262 is 175, key is test_row_0/A:col10/1733617028882/Put/seqid=0 2024-12-08T00:17:09,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742418_1594 (size=31255) 2024-12-08T00:17:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T00:17:09,907 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/1ccc96541ab742d3b8a4e72c8e572262 2024-12-08T00:17:09,919 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8dce527699794c11973236493befcce7 is 50, key is test_row_0/B:col10/1733617028882/Put/seqid=0 2024-12-08T00:17:09,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742419_1595 (size=12301) 2024-12-08T00:17:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T00:17:10,324 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8dce527699794c11973236493befcce7 2024-12-08T00:17:10,341 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/095c03b4aa66440cb22530788daf71f3 is 50, key is test_row_0/C:col10/1733617028882/Put/seqid=0 2024-12-08T00:17:10,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742420_1596 (size=12301) 2024-12-08T00:17:10,745 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/095c03b4aa66440cb22530788daf71f3 2024-12-08T00:17:10,755 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/A/1ccc96541ab742d3b8a4e72c8e572262 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/1ccc96541ab742d3b8a4e72c8e572262 2024-12-08T00:17:10,759 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/1ccc96541ab742d3b8a4e72c8e572262, entries=150, sequenceid=358, filesize=30.5 K 2024-12-08T00:17:10,759 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/B/8dce527699794c11973236493befcce7 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8dce527699794c11973236493befcce7 2024-12-08T00:17:10,763 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8dce527699794c11973236493befcce7, entries=150, sequenceid=358, filesize=12.0 K 2024-12-08T00:17:10,763 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/.tmp/C/095c03b4aa66440cb22530788daf71f3 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/095c03b4aa66440cb22530788daf71f3 2024-12-08T00:17:10,767 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/095c03b4aa66440cb22530788daf71f3, entries=150, sequenceid=358, filesize=12.0 K 2024-12-08T00:17:10,768 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 951f2f04d16f20e5ba0261e33c9a4276 in 1691ms, sequenceid=358, compaction requested=true 2024-12-08T00:17:10,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/735b202a01cf440b92bde89a23342c88, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2846d10135234f99849050eeee0d4bc1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/894706054cf749c9ad46f0f8c9fb447d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/233b3d5bf5ee4427b4dfd8bfd50d5e36, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/73e9c64fd0024a53b695898ff5fdc9ae, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/27e80f247e084f53999527af03e34f7a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/913a23e7f329403e8f47d707cace20ff, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2e719d416e0e4a29b45f8c7b69549942, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/6651176560cd4f42a645537385bf73f7, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/80ebe9a29e3a49318418fc61063bf9fe, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/707491653f864c0f8ece2d6032c0cf53, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/e72b485e4d274bd99d9a558611ef8bff, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/74a140fd6e714679a21266b497500f92, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/0a4e327f478f45d7bcf7db782ac7ab2a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/024daa4a07a04989b87330fda6e3d078, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5dac596401604a9ba5156c57f136754b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/531c7ddc60f040abaced466e9f4ba79d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5b30981ed74444e8980b2a3360caa567, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/38b3766c657243b0848b14b9e48f262b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/88cc8ffec1fc43d2884b934ea4c910cb] to archive 2024-12-08T00:17:10,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:17:10,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/735b202a01cf440b92bde89a23342c88 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/735b202a01cf440b92bde89a23342c88 2024-12-08T00:17:10,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2846d10135234f99849050eeee0d4bc1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2846d10135234f99849050eeee0d4bc1 2024-12-08T00:17:10,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/894706054cf749c9ad46f0f8c9fb447d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/894706054cf749c9ad46f0f8c9fb447d 2024-12-08T00:17:10,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/233b3d5bf5ee4427b4dfd8bfd50d5e36 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/233b3d5bf5ee4427b4dfd8bfd50d5e36 2024-12-08T00:17:10,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/73e9c64fd0024a53b695898ff5fdc9ae to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/73e9c64fd0024a53b695898ff5fdc9ae 2024-12-08T00:17:10,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/27e80f247e084f53999527af03e34f7a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/27e80f247e084f53999527af03e34f7a 2024-12-08T00:17:10,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/913a23e7f329403e8f47d707cace20ff to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/913a23e7f329403e8f47d707cace20ff 2024-12-08T00:17:10,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2e719d416e0e4a29b45f8c7b69549942 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/2e719d416e0e4a29b45f8c7b69549942 2024-12-08T00:17:10,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/6651176560cd4f42a645537385bf73f7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/6651176560cd4f42a645537385bf73f7 2024-12-08T00:17:10,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/80ebe9a29e3a49318418fc61063bf9fe to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/80ebe9a29e3a49318418fc61063bf9fe 2024-12-08T00:17:10,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/707491653f864c0f8ece2d6032c0cf53 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/707491653f864c0f8ece2d6032c0cf53 2024-12-08T00:17:10,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/e72b485e4d274bd99d9a558611ef8bff to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/e72b485e4d274bd99d9a558611ef8bff 2024-12-08T00:17:10,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/74a140fd6e714679a21266b497500f92 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/74a140fd6e714679a21266b497500f92 2024-12-08T00:17:10,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/0a4e327f478f45d7bcf7db782ac7ab2a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/0a4e327f478f45d7bcf7db782ac7ab2a 2024-12-08T00:17:10,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/024daa4a07a04989b87330fda6e3d078 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/024daa4a07a04989b87330fda6e3d078 2024-12-08T00:17:10,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5dac596401604a9ba5156c57f136754b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5dac596401604a9ba5156c57f136754b 2024-12-08T00:17:10,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/531c7ddc60f040abaced466e9f4ba79d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/531c7ddc60f040abaced466e9f4ba79d 2024-12-08T00:17:10,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5b30981ed74444e8980b2a3360caa567 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/5b30981ed74444e8980b2a3360caa567 2024-12-08T00:17:10,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/38b3766c657243b0848b14b9e48f262b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/38b3766c657243b0848b14b9e48f262b 2024-12-08T00:17:10,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/88cc8ffec1fc43d2884b934ea4c910cb to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/88cc8ffec1fc43d2884b934ea4c910cb 2024-12-08T00:17:10,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a5eebd7f5bad4ac286dfeef44e3f44a2, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a47d1124c40046cf8a88d595c63c6205, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ca522ffe0d5646c498173fa8c2647b72, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ee6ea77cd47548a490940a7b0910d0e6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/0ea97b8a88d244ae8e806db9530b24d6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/5b10977367094da6be81b3c2f22acde1, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e6691627fcad41f1b0951b5e59284ad4, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/49a55d319cda4396a48c88f7e822526d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/f15795d240d44ea996d844ffaff759ab, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/2cf208c15e204325a46b3e3baba54bc3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/3722f3bb5c2b48d59232763614fdb560, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/28785985a1c44bed9d9f8d3af1b2379d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/024ac240564443b987542ef6b545c269, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/109fb6ea0adb4fe08ef08922e5937e6f, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/797be330beda4d52a03d3cbe05c96a29, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8b88fe0dae5646b9b2c3bca8a6b6216b, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8237ee88cbb348c1894a15148f73eb64, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a9c09d68a1ce48e18be0203e79ece9c4, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/d1c851a7fa084abcb229d1a238acb6c3, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/69bbafaffd684fbd81f918ad4747a16f] to archive 2024-12-08T00:17:10,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:17:10,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a5eebd7f5bad4ac286dfeef44e3f44a2 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a5eebd7f5bad4ac286dfeef44e3f44a2 2024-12-08T00:17:10,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a47d1124c40046cf8a88d595c63c6205 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a47d1124c40046cf8a88d595c63c6205 2024-12-08T00:17:10,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ca522ffe0d5646c498173fa8c2647b72 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ca522ffe0d5646c498173fa8c2647b72 2024-12-08T00:17:10,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ee6ea77cd47548a490940a7b0910d0e6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/ee6ea77cd47548a490940a7b0910d0e6 2024-12-08T00:17:10,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/0ea97b8a88d244ae8e806db9530b24d6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/0ea97b8a88d244ae8e806db9530b24d6 2024-12-08T00:17:10,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/5b10977367094da6be81b3c2f22acde1 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/5b10977367094da6be81b3c2f22acde1 2024-12-08T00:17:10,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e6691627fcad41f1b0951b5e59284ad4 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e6691627fcad41f1b0951b5e59284ad4 2024-12-08T00:17:10,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/49a55d319cda4396a48c88f7e822526d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/49a55d319cda4396a48c88f7e822526d 2024-12-08T00:17:10,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/f15795d240d44ea996d844ffaff759ab to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/f15795d240d44ea996d844ffaff759ab 2024-12-08T00:17:10,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/2cf208c15e204325a46b3e3baba54bc3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/2cf208c15e204325a46b3e3baba54bc3 2024-12-08T00:17:10,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/3722f3bb5c2b48d59232763614fdb560 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/3722f3bb5c2b48d59232763614fdb560 2024-12-08T00:17:10,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/28785985a1c44bed9d9f8d3af1b2379d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/28785985a1c44bed9d9f8d3af1b2379d 2024-12-08T00:17:10,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/024ac240564443b987542ef6b545c269 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/024ac240564443b987542ef6b545c269 2024-12-08T00:17:10,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/109fb6ea0adb4fe08ef08922e5937e6f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/109fb6ea0adb4fe08ef08922e5937e6f 2024-12-08T00:17:10,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/797be330beda4d52a03d3cbe05c96a29 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/797be330beda4d52a03d3cbe05c96a29 2024-12-08T00:17:10,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8b88fe0dae5646b9b2c3bca8a6b6216b to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8b88fe0dae5646b9b2c3bca8a6b6216b 2024-12-08T00:17:10,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8237ee88cbb348c1894a15148f73eb64 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8237ee88cbb348c1894a15148f73eb64 2024-12-08T00:17:10,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a9c09d68a1ce48e18be0203e79ece9c4 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/a9c09d68a1ce48e18be0203e79ece9c4 2024-12-08T00:17:10,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/d1c851a7fa084abcb229d1a238acb6c3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/d1c851a7fa084abcb229d1a238acb6c3 2024-12-08T00:17:10,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/69bbafaffd684fbd81f918ad4747a16f to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/69bbafaffd684fbd81f918ad4747a16f 2024-12-08T00:17:10,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d730e278ecf343ec93d2e2796716ceaa, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d26a8086bbee447d9eff7b8f8e347b6d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d414af87bc12486a93f524554b362b64, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e0cc6fbf3b7541f7850072afe8cf7633, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/16061dc79eef44839a9c8b57c212d998, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/00add05daa65473c93dfca57c4f29444, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/ef73429da54f411bac12c954652701c4, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/c7c65409aa4e4804b8379a019260c96d, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/1b85e17884ee4be085ed602484b8c593, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/aa864c38cc9c4448a1812e3adace63f9, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/6257d39706824bfd849f73fac1409ddb, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/396ca0f236dd42f681d913b18e36e154, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/61dbb714fb9d487a812d6076c790a1d6, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a933b64ad4254064b104fba7e5205d87, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/b54d76607a6f400f9f38bc4a52168795, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/fbb6683ab89542889e79a1e8595f2761, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e9632f29c8664f8f8642827b4a866b0a, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/25346b1818c542499e8685c80635a24e, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a8d048e1e6104130ac45751d6ccdf867, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/4a4bf4d1bba5417b80f2a812f506920d] to archive 2024-12-08T00:17:10,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T00:17:10,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d730e278ecf343ec93d2e2796716ceaa to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d730e278ecf343ec93d2e2796716ceaa 2024-12-08T00:17:10,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d26a8086bbee447d9eff7b8f8e347b6d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d26a8086bbee447d9eff7b8f8e347b6d 2024-12-08T00:17:10,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d414af87bc12486a93f524554b362b64 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/d414af87bc12486a93f524554b362b64 2024-12-08T00:17:10,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e0cc6fbf3b7541f7850072afe8cf7633 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e0cc6fbf3b7541f7850072afe8cf7633 2024-12-08T00:17:10,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/16061dc79eef44839a9c8b57c212d998 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/16061dc79eef44839a9c8b57c212d998 2024-12-08T00:17:10,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/00add05daa65473c93dfca57c4f29444 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/00add05daa65473c93dfca57c4f29444 2024-12-08T00:17:10,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/ef73429da54f411bac12c954652701c4 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/ef73429da54f411bac12c954652701c4 2024-12-08T00:17:10,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/c7c65409aa4e4804b8379a019260c96d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/c7c65409aa4e4804b8379a019260c96d 2024-12-08T00:17:10,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/1b85e17884ee4be085ed602484b8c593 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/1b85e17884ee4be085ed602484b8c593 2024-12-08T00:17:10,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/aa864c38cc9c4448a1812e3adace63f9 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/aa864c38cc9c4448a1812e3adace63f9 2024-12-08T00:17:10,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/6257d39706824bfd849f73fac1409ddb to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/6257d39706824bfd849f73fac1409ddb 2024-12-08T00:17:10,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/396ca0f236dd42f681d913b18e36e154 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/396ca0f236dd42f681d913b18e36e154 2024-12-08T00:17:10,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/61dbb714fb9d487a812d6076c790a1d6 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/61dbb714fb9d487a812d6076c790a1d6 2024-12-08T00:17:10,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a933b64ad4254064b104fba7e5205d87 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a933b64ad4254064b104fba7e5205d87 2024-12-08T00:17:10,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/b54d76607a6f400f9f38bc4a52168795 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/b54d76607a6f400f9f38bc4a52168795 2024-12-08T00:17:10,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/fbb6683ab89542889e79a1e8595f2761 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/fbb6683ab89542889e79a1e8595f2761 2024-12-08T00:17:10,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e9632f29c8664f8f8642827b4a866b0a to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/e9632f29c8664f8f8642827b4a866b0a 2024-12-08T00:17:10,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/25346b1818c542499e8685c80635a24e to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/25346b1818c542499e8685c80635a24e 2024-12-08T00:17:10,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a8d048e1e6104130ac45751d6ccdf867 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/a8d048e1e6104130ac45751d6ccdf867 2024-12-08T00:17:10,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/4a4bf4d1bba5417b80f2a812f506920d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/4a4bf4d1bba5417b80f2a812f506920d 2024-12-08T00:17:10,848 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/recovered.edits/361.seqid, newMaxSeqId=361, maxSeqId=4 2024-12-08T00:17:10,848 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. 2024-12-08T00:17:10,848 DEBUG [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 951f2f04d16f20e5ba0261e33c9a4276: Waiting for close lock at 1733617029075Running coprocessor pre-close hooks at 1733617029075Disabling compacts and flushes for region at 1733617029075Disabling writes for close at 1733617029076 (+1 ms)Obtaining lock to block concurrent updates at 1733617029076Preparing flush snapshotting stores in 951f2f04d16f20e5ba0261e33c9a4276 at 1733617029076Finished memstore snapshotting TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733617029077 (+1 ms)Flushing stores of TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276. at 1733617029078 (+1 ms)Flushing 951f2f04d16f20e5ba0261e33c9a4276/A: creating writer at 1733617029079 (+1 ms)Flushing mob file 951f2f04d16f20e5ba0261e33c9a4276/A: appending metadata at 1733617029085 (+6 ms)Flushing mob file 951f2f04d16f20e5ba0261e33c9a4276/A: closing flushed file at 1733617029085Flushing 951f2f04d16f20e5ba0261e33c9a4276/A: appending metadata at 1733617029500 (+415 ms)Flushing 951f2f04d16f20e5ba0261e33c9a4276/A: closing flushed file at 1733617029500Flushing 951f2f04d16f20e5ba0261e33c9a4276/B: creating writer at 1733617029916 (+416 ms)Flushing 951f2f04d16f20e5ba0261e33c9a4276/B: appending metadata at 1733617029919 (+3 ms)Flushing 951f2f04d16f20e5ba0261e33c9a4276/B: closing flushed file at 1733617029919Flushing 951f2f04d16f20e5ba0261e33c9a4276/C: creating writer at 1733617030334 (+415 ms)Flushing 951f2f04d16f20e5ba0261e33c9a4276/C: appending metadata at 1733617030336 (+2 ms)Flushing 951f2f04d16f20e5ba0261e33c9a4276/C: closing flushed file at 1733617030336Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5530f6cf: reopening flushed file at 1733617030754 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19da0dc1: reopening flushed file at 1733617030759 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ac27e54: reopening flushed file at 1733617030763 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 951f2f04d16f20e5ba0261e33c9a4276 in 1691ms, sequenceid=358, compaction requested=true at 1733617030768 (+5 ms)Writing region close event to WAL at 1733617030845 (+77 ms)Running coprocessor post-close hooks at 1733617030848 (+3 ms)Closed at 1733617030848 2024-12-08T00:17:10,850 INFO [RS_CLOSE_REGION-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:10,851 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=951f2f04d16f20e5ba0261e33c9a4276, regionState=CLOSED 2024-12-08T00:17:10,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 because future has completed 2024-12-08T00:17:10,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-08T00:17:10,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 951f2f04d16f20e5ba0261e33c9a4276, server=c1a8cf8d4cfa,36021,1733616847970 in 1.9310 sec 2024-12-08T00:17:10,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-12-08T00:17:10,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951f2f04d16f20e5ba0261e33c9a4276, UNASSIGN in 1.9380 sec 2024-12-08T00:17:10,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-08T00:17:10,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9410 sec 2024-12-08T00:17:10,858 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733617030858"}]},"ts":"1733617030858"} 2024-12-08T00:17:10,860 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T00:17:10,860 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T00:17:10,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9540 sec 2024-12-08T00:17:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T00:17:11,071 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:17:11,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T00:17:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:17:11,074 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:17:11,075 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:17:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-08T00:17:11,076 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,078 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C, FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/recovered.edits] 2024-12-08T00:17:11,079 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/1ccc96541ab742d3b8a4e72c8e572262 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/1ccc96541ab742d3b8a4e72c8e572262 2024-12-08T00:17:11,080 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/af5895b99ece459e962710cf40fcda4d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/af5895b99ece459e962710cf40fcda4d 2024-12-08T00:17:11,082 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/f97543c7bbb7428690334bc7e6e8f799 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/A/f97543c7bbb7428690334bc7e6e8f799 2024-12-08T00:17:11,084 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8dce527699794c11973236493befcce7 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8dce527699794c11973236493befcce7 2024-12-08T00:17:11,085 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8fd3c2b98f0e4e5f8264a0e173554e7d to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/8fd3c2b98f0e4e5f8264a0e173554e7d 2024-12-08T00:17:11,085 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e9c03bcd002041019b90e4a7874f7f80 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/B/e9c03bcd002041019b90e4a7874f7f80 2024-12-08T00:17:11,087 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/095c03b4aa66440cb22530788daf71f3 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/095c03b4aa66440cb22530788daf71f3 2024-12-08T00:17:11,088 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/30dc484fed2749e8a3d83b8a38ba4c45 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/30dc484fed2749e8a3d83b8a38ba4c45 2024-12-08T00:17:11,089 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/86361b15f1ea48338152ee483472d321 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/C/86361b15f1ea48338152ee483472d321 2024-12-08T00:17:11,091 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/recovered.edits/361.seqid to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276/recovered.edits/361.seqid 2024-12-08T00:17:11,092 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/default/TestAcidGuarantees/951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,092 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T00:17:11,092 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T00:17:11,093 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T00:17:11,096 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080311898b7fba47ebb1c81a23687d2989_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080311898b7fba47ebb1c81a23687d2989_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,097 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208270cdf74999a452f928bdd9d86fca052_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208270cdf74999a452f928bdd9d86fca052_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,098 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083fd7ee9290b34172aa7b85c01255d3cd_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083fd7ee9290b34172aa7b85c01255d3cd_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,099 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120847c69d11700f4137befafff0bebc7832_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120847c69d11700f4137befafff0bebc7832_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,101 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084ad2fb73b6e2431a8e28c8e611993013_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084ad2fb73b6e2431a8e28c8e611993013_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,102 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120855c5a9ed41914c3295a54add3292d837_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120855c5a9ed41914c3295a54add3292d837_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,103 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120861ec48dfc4c34408a3375d4e879e4cf4_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120861ec48dfc4c34408a3375d4e879e4cf4_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,105 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208659eebd993ac439ebfe2a375a526c65f_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208659eebd993ac439ebfe2a375a526c65f_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,106 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208875f0a59006f43d292d9d8884a56ce56_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208875f0a59006f43d292d9d8884a56ce56_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,107 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120887bcf32b8cb0488e942d913e47c807a6_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120887bcf32b8cb0488e942d913e47c807a6_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,108 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088c7b2a03003c46909041384950d77aa8_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088c7b2a03003c46909041384950d77aa8_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,109 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208960044b3abda4e34a255b7819397ff9d_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208960044b3abda4e34a255b7819397ff9d_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,110 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a0aba96ae3cd4cce88983f2db84ddbf1_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a0aba96ae3cd4cce88983f2db84ddbf1_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,111 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b2dc2c97df5845e6a6f2557ee6053ea2_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b2dc2c97df5845e6a6f2557ee6053ea2_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,113 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d6345fbc4b62405f9138e95a4f924e88_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d6345fbc4b62405f9138e95a4f924e88_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,114 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dbc5f93143294853b295058853a56172_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dbc5f93143294853b295058853a56172_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,115 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e1409b28e6cf418cb9bd513905df4cd8_951f2f04d16f20e5ba0261e33c9a4276 to hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e1409b28e6cf418cb9bd513905df4cd8_951f2f04d16f20e5ba0261e33c9a4276 2024-12-08T00:17:11,115 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T00:17:11,118 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:17:11,120 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T00:17:11,122 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T00:17:11,123 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:17:11,123 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T00:17:11,124 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733617031124"}]},"ts":"9223372036854775807"} 2024-12-08T00:17:11,126 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T00:17:11,126 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 951f2f04d16f20e5ba0261e33c9a4276, NAME => 'TestAcidGuarantees,,1733616996781.951f2f04d16f20e5ba0261e33c9a4276.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T00:17:11,126 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T00:17:11,127 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733617031126"}]},"ts":"9223372036854775807"} 2024-12-08T00:17:11,129 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T00:17:11,129 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T00:17:11,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-12-08T00:17:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-08T00:17:11,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T00:17:11,182 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T00:17:11,189 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 252), OpenFileDescriptor=445 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=382 (was 406), ProcessCount=11 (was 11), AvailableMemoryMB=8490 (was 8927) 2024-12-08T00:17:11,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T00:17:11,190 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T00:17:11,190 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T00:17:11,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:11,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:11,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T00:17:11,190 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T00:17:11,190 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-12-08T00:17:11,190 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c1a8cf8d4cfa,46259,1733616847193 2024-12-08T00:17:11,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T00:17:11,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T00:17:11,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:17:11,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:17:11,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T00:17:11,255 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T00:17:11,255 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T00:17:11,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:11,256 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T00:17:11,256 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T00:17:11,256 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c1a8cf8d4cfa,36021,1733616847970' ***** 2024-12-08T00:17:11,256 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T00:17:11,256 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T00:17:11,257 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T00:17:11,257 INFO [RS:0;c1a8cf8d4cfa:36021 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T00:17:11,258 INFO [RS:0;c1a8cf8d4cfa:36021 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T00:17:11,258 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(959): stopping server c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:17:11,258 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T00:17:11,258 INFO [RS:0;c1a8cf8d4cfa:36021 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c1a8cf8d4cfa:36021. 2024-12-08T00:17:11,258 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T00:17:11,258 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:11,258 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T00:17:11,258 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T00:17:11,258 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T00:17:11,258 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T00:17:11,259 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T00:17:11,259 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T00:17:11,259 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T00:17:11,259 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T00:17:11,259 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T00:17:11,259 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T00:17:11,259 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T00:17:11,259 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T00:17:11,259 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-08T00:17:11,277 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/info/7d7eb1da46d84eb5a67e43c4d2cce442 is 95, key is TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068./info:/1733616877102/DeleteFamily/seqid=0 2024-12-08T00:17:11,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742421_1597 (size=6514) 2024-12-08T00:17:11,312 INFO [regionserver/c1a8cf8d4cfa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T00:17:11,459 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T00:17:11,659 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T00:17:11,682 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/info/7d7eb1da46d84eb5a67e43c4d2cce442 2024-12-08T00:17:11,698 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/ns/8939ac9059ed420994f02faa3a5afd5a is 93, key is TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068./ns:/1733616877067/DeleteFamily/seqid=0 2024-12-08T00:17:11,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742422_1598 (size=5906) 2024-12-08T00:17:11,859 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T00:17:12,059 INFO [regionserver/c1a8cf8d4cfa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-08T00:17:12,059 INFO [regionserver/c1a8cf8d4cfa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-08T00:17:12,060 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T00:17:12,101 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/ns/8939ac9059ed420994f02faa3a5afd5a 2024-12-08T00:17:12,117 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/rep_barrier/0fbd12314dd84d9dabf69340df7cd234 is 102, key is TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068./rep_barrier:/1733616877067/DeleteFamily/seqid=0 2024-12-08T00:17:12,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742423_1599 (size=6025) 2024-12-08T00:17:12,260 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T00:17:12,460 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T00:17:12,520 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/rep_barrier/0fbd12314dd84d9dabf69340df7cd234 2024-12-08T00:17:12,537 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/table/bb5ec631d7dd4d69992ff50f08369e2e is 96, key is TestAcidGuarantees,,1733616851646.9938c46f9ca7a3abf34703707edea068./table:/1733616877067/DeleteFamily/seqid=0 2024-12-08T00:17:12,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742424_1600 (size=5936) 2024-12-08T00:17:12,660 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T00:17:12,860 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T00:17:12,941 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/table/bb5ec631d7dd4d69992ff50f08369e2e 2024-12-08T00:17:12,949 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/info/7d7eb1da46d84eb5a67e43c4d2cce442 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/info/7d7eb1da46d84eb5a67e43c4d2cce442 2024-12-08T00:17:12,953 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/info/7d7eb1da46d84eb5a67e43c4d2cce442, entries=12, sequenceid=90, filesize=6.4 K 2024-12-08T00:17:12,954 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/ns/8939ac9059ed420994f02faa3a5afd5a as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/ns/8939ac9059ed420994f02faa3a5afd5a 2024-12-08T00:17:12,959 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/ns/8939ac9059ed420994f02faa3a5afd5a, entries=8, sequenceid=90, filesize=5.8 K 2024-12-08T00:17:12,960 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/rep_barrier/0fbd12314dd84d9dabf69340df7cd234 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/rep_barrier/0fbd12314dd84d9dabf69340df7cd234 2024-12-08T00:17:12,964 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/rep_barrier/0fbd12314dd84d9dabf69340df7cd234, entries=6, sequenceid=90, filesize=5.9 K 2024-12-08T00:17:12,965 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/.tmp/table/bb5ec631d7dd4d69992ff50f08369e2e as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/table/bb5ec631d7dd4d69992ff50f08369e2e 2024-12-08T00:17:12,968 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/table/bb5ec631d7dd4d69992ff50f08369e2e, entries=7, sequenceid=90, filesize=5.8 K 2024-12-08T00:17:12,968 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false 2024-12-08T00:17:12,973 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-08T00:17:12,973 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T00:17:12,974 INFO [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T00:17:12,974 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733617031259Running coprocessor pre-close hooks at 1733617031259Disabling compacts and flushes for region at 1733617031259Disabling writes for close at 1733617031259Obtaining lock to block concurrent updates at 1733617031259Preparing flush snapshotting stores in 1588230740 at 1733617031259Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733617031259Flushing stores of hbase:meta,,1.1588230740 at 1733617031260 (+1 ms)Flushing 1588230740/info: creating writer at 1733617031260Flushing 1588230740/info: appending metadata at 1733617031276 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733617031276Flushing 1588230740/ns: creating writer at 1733617031685 (+409 ms)Flushing 1588230740/ns: appending metadata at 1733617031698 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733617031698Flushing 1588230740/rep_barrier: creating writer at 1733617032104 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733617032116 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733617032116Flushing 1588230740/table: creating writer at 1733617032524 (+408 ms)Flushing 1588230740/table: appending metadata at 1733617032537 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733617032537Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fab517b: reopening flushed file at 1733617032948 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c2090b2: reopening flushed file at 1733617032954 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@658751ae: reopening flushed file at 1733617032959 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b83bd3e: reopening flushed file at 1733617032964 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false at 1733617032968 (+4 ms)Writing region close event to WAL at 1733617032970 (+2 ms)Running coprocessor post-close hooks at 1733617032973 (+3 ms)Closed at 1733617032974 (+1 ms) 2024-12-08T00:17:12,974 DEBUG [RS_CLOSE_META-regionserver/c1a8cf8d4cfa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T00:17:13,061 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(976): stopping server c1a8cf8d4cfa,36021,1733616847970; all regions closed. 2024-12-08T00:17:13,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741834_1010 (size=25344) 2024-12-08T00:17:13,067 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/oldWALs 2024-12-08T00:17:13,067 INFO [RS:0;c1a8cf8d4cfa:36021 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c1a8cf8d4cfa%2C36021%2C1733616847970.meta:.meta(num 1733616851187) 2024-12-08T00:17:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741832_1008 (size=17931305) 2024-12-08T00:17:13,071 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/oldWALs 2024-12-08T00:17:13,071 INFO [RS:0;c1a8cf8d4cfa:36021 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c1a8cf8d4cfa%2C36021%2C1733616847970:(num 1733616850180) 2024-12-08T00:17:13,071 DEBUG [RS:0;c1a8cf8d4cfa:36021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T00:17:13,071 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T00:17:13,071 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T00:17:13,071 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.ChoreService(370): Chore service for: regionserver/c1a8cf8d4cfa:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T00:17:13,071 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T00:17:13,071 INFO [regionserver/c1a8cf8d4cfa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T00:17:13,072 INFO [RS:0;c1a8cf8d4cfa:36021 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36021 2024-12-08T00:17:13,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T00:17:13,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c1a8cf8d4cfa,36021,1733616847970 2024-12-08T00:17:13,335 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T00:17:13,421 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c1a8cf8d4cfa,36021,1733616847970] 2024-12-08T00:17:13,430 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c1a8cf8d4cfa,36021,1733616847970 already deleted, retry=false 2024-12-08T00:17:13,430 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c1a8cf8d4cfa,36021,1733616847970 expired; onlineServers=0 2024-12-08T00:17:13,430 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c1a8cf8d4cfa,46259,1733616847193' ***** 2024-12-08T00:17:13,430 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T00:17:13,430 INFO [M:0;c1a8cf8d4cfa:46259 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T00:17:13,430 INFO [M:0;c1a8cf8d4cfa:46259 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T00:17:13,430 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T00:17:13,430 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T00:17:13,430 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T00:17:13,430 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster-HFileCleaner.large.0-1733616849870 {}] cleaner.HFileCleaner(306): Exit Thread[master/c1a8cf8d4cfa:0:becomeActiveMaster-HFileCleaner.large.0-1733616849870,5,FailOnTimeoutGroup] 2024-12-08T00:17:13,430 DEBUG [master/c1a8cf8d4cfa:0:becomeActiveMaster-HFileCleaner.small.0-1733616849870 {}] cleaner.HFileCleaner(306): Exit Thread[master/c1a8cf8d4cfa:0:becomeActiveMaster-HFileCleaner.small.0-1733616849870,5,FailOnTimeoutGroup] 2024-12-08T00:17:13,431 INFO [M:0;c1a8cf8d4cfa:46259 {}] hbase.ChoreService(370): Chore service for: master/c1a8cf8d4cfa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T00:17:13,431 INFO [M:0;c1a8cf8d4cfa:46259 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T00:17:13,431 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] master.HMaster(1795): Stopping service threads 2024-12-08T00:17:13,431 INFO [M:0;c1a8cf8d4cfa:46259 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T00:17:13,431 INFO [M:0;c1a8cf8d4cfa:46259 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T00:17:13,431 ERROR [M:0;c1a8cf8d4cfa:46259 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-08T00:17:13,431 INFO [M:0;c1a8cf8d4cfa:46259 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T00:17:13,431 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T00:17:13,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T00:17:13,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T00:17:13,451 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-08T00:17:13,451 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] master.ActiveMasterManager(353): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-08T00:17:13,451 INFO [M:0;c1a8cf8d4cfa:46259 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/.lastflushedseqids 2024-12-08T00:17:13,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742425_1601 (size=72) 2024-12-08T00:17:13,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T00:17:13,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36021-0x10002c5ccce0001, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T00:17:13,520 INFO [RS:0;c1a8cf8d4cfa:36021 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T00:17:13,521 INFO [RS:0;c1a8cf8d4cfa:36021 {}] regionserver.HRegionServer(1031): Exiting; stopping=c1a8cf8d4cfa,36021,1733616847970; zookeeper connection closed. 2024-12-08T00:17:13,521 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e58566e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e58566e 2024-12-08T00:17:13,522 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-08T00:17:13,862 INFO [M:0;c1a8cf8d4cfa:46259 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T00:17:13,862 INFO [M:0;c1a8cf8d4cfa:46259 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T00:17:13,862 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T00:17:13,862 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T00:17:13,863 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T00:17:13,863 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T00:17:13,863 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T00:17:13,863 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=755.03 KB heapSize=923.56 KB 2024-12-08T00:17:13,884 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/27307ea9573e4551bdca557971e0c0ad is 82, key is hbase:meta,,1/info:regioninfo/1733616851277/Put/seqid=0 2024-12-08T00:17:13,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742426_1602 (size=5672) 2024-12-08T00:17:14,314 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2076 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/27307ea9573e4551bdca557971e0c0ad 2024-12-08T00:17:14,333 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ffb09d1374d14224b7672072a71aa544 is 2479, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1733616880283/Put/seqid=0 2024-12-08T00:17:14,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742427_1603 (size=44997) 2024-12-08T00:17:14,738 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754.48 KB at sequenceid=2076 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ffb09d1374d14224b7672072a71aa544 2024-12-08T00:17:14,744 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ffb09d1374d14224b7672072a71aa544 2024-12-08T00:17:14,761 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/981cc03e6aca43b4aa30b3e36a860693 is 69, key is c1a8cf8d4cfa,36021,1733616847970/rs:state/1733616849946/Put/seqid=0 2024-12-08T00:17:14,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073742428_1604 (size=5156) 2024-12-08T00:17:15,164 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2076 (bloomFilter=true), to=hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/981cc03e6aca43b4aa30b3e36a860693 2024-12-08T00:17:15,168 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/27307ea9573e4551bdca557971e0c0ad as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/27307ea9573e4551bdca557971e0c0ad 2024-12-08T00:17:15,171 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/27307ea9573e4551bdca557971e0c0ad, entries=8, sequenceid=2076, filesize=5.5 K 2024-12-08T00:17:15,172 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ffb09d1374d14224b7672072a71aa544 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ffb09d1374d14224b7672072a71aa544 2024-12-08T00:17:15,175 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ffb09d1374d14224b7672072a71aa544 2024-12-08T00:17:15,175 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ffb09d1374d14224b7672072a71aa544, entries=168, sequenceid=2076, filesize=43.9 K 2024-12-08T00:17:15,175 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/981cc03e6aca43b4aa30b3e36a860693 as hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/981cc03e6aca43b4aa30b3e36a860693 2024-12-08T00:17:15,178 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38367/user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/981cc03e6aca43b4aa30b3e36a860693, entries=1, sequenceid=2076, filesize=5.0 K 2024-12-08T00:17:15,179 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegion(3140): Finished flush of dataSize ~755.03 KB/773152, heapSize ~923.27 KB/945424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1316ms, sequenceid=2076, compaction requested=false 2024-12-08T00:17:15,181 INFO [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T00:17:15,181 DEBUG [M:0;c1a8cf8d4cfa:46259 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733617033862Disabling compacts and flushes for region at 1733617033862Disabling writes for close at 1733617033863 (+1 ms)Obtaining lock to block concurrent updates at 1733617033863Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733617033863Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=773152, getHeapSize=945664, getOffHeapSize=0, getCellsCount=2269 at 1733617033863Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733617033865 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733617033865Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733617033884 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733617033884Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733617034318 (+434 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733617034333 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733617034333Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733617034744 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733617034760 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733617034760Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6caea153: reopening flushed file at 1733617035168 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54ba4d8a: reopening flushed file at 1733617035171 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67bfea74: reopening flushed file at 1733617035175 (+4 ms)Finished flush of dataSize ~755.03 KB/773152, heapSize ~923.27 KB/945424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1316ms, sequenceid=2076, compaction requested=false at 1733617035179 (+4 ms)Writing region close event to WAL at 1733617035181 (+2 ms)Closed at 1733617035181 2024-12-08T00:17:15,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36767 is added to blk_1073741830_1006 (size=909283) 2024-12-08T00:17:15,184 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/05269d59-b1ca-d621-5a9e-f169e5c5d329/MasterData/WALs/c1a8cf8d4cfa,46259,1733616847193/c1a8cf8d4cfa%2C46259%2C1733616847193.1733616849238 not finished, retry = 0 2024-12-08T00:17:15,285 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T00:17:15,285 INFO [M:0;c1a8cf8d4cfa:46259 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T00:17:15,286 INFO [M:0;c1a8cf8d4cfa:46259 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46259 2024-12-08T00:17:15,286 INFO [M:0;c1a8cf8d4cfa:46259 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T00:17:15,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T00:17:15,403 INFO [M:0;c1a8cf8d4cfa:46259 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T00:17:15,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46259-0x10002c5ccce0000, quorum=127.0.0.1:60060, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T00:17:15,410 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T00:17:15,413 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T00:17:15,413 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T00:17:15,413 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T00:17:15,413 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/hadoop.log.dir/,STOPPED} 2024-12-08T00:17:15,416 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T00:17:15,416 WARN [BP-1600602867-172.17.0.2-1733616842832 heartbeating to localhost/127.0.0.1:38367 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T00:17:15,416 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T00:17:15,416 WARN [BP-1600602867-172.17.0.2-1733616842832 heartbeating to localhost/127.0.0.1:38367 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1600602867-172.17.0.2-1733616842832 (Datanode Uuid 99912f89-3e06-4776-a0b9-bcdf0d55620a) service to localhost/127.0.0.1:38367 2024-12-08T00:17:15,418 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/cluster_da41c416-9640-b4a5-e754-4cc4f92f286a/data/data1/current/BP-1600602867-172.17.0.2-1733616842832 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T00:17:15,419 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/cluster_da41c416-9640-b4a5-e754-4cc4f92f286a/data/data2/current/BP-1600602867-172.17.0.2-1733616842832 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T00:17:15,419 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T00:17:15,426 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T00:17:15,427 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T00:17:15,427 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T00:17:15,427 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T00:17:15,427 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/34a1f1ed-dcfb-1550-2cc7-13de078c8042/hadoop.log.dir/,STOPPED} 2024-12-08T00:17:15,441 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T00:17:15,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down